fix(eval): update openapi spec

.github/actions/changes/action.yaml

@@ -73,6 +73,15 @@ runs:
           openapi:
             - 'openapi/**'
             - 'plugins/*/openapi/**'
+            - 'plugins/*/src/**'
+            - 'plugins/*/pyproject.toml'
+            - 'packages/nmp_common/src/nmp_common/api/**'
+            - 'packages/nmp_common/src/nmp_common/datamodel/**'
+            - 'packages/nemo_platform_plugin/src/**'
+            - 'packages/nemo_evaluator_sdk/src/**'
+            - 'script/generate_openapi_spec.py'
+            - 'script/generate-openapi-spec.sh'
+            - 'script/openapi_helper/**'
           tests:
             - 'tests/**'
             - 'pytest.ini'

.github/workflows/ci.yaml

@@ -787,6 +787,8 @@ jobs:
           enable-cache: true
           python-version: "3.12"
           cache-dependency-glob: uv.lock
+      - name: Check uv lockfile
+        run: uv lock --check
       - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
           node-version: '22'
@@ -1628,6 +1630,8 @@ jobs:
         run: pnpm --filter @nemo/sdk gen:all-force
       - name: Typecheck SDK generated output
         run: pnpm --filter @nemo/sdk typecheck
+      - name: Typecheck web packages against regenerated SDK
+        run: pnpm run --recursive --parallel --if-present typecheck
 
   web-studio-deps:
     name: Web studio deps check

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py

@@ -295,7 +295,7 @@ class AggregateScoreBase(BaseModel):
     name: str = Field(description="Name of the score.")
     count: int | None = Field(
         default=None,
-        description="Number of samples evaluated (excluding NaN). None when the sample size is unknown "
+        description="Number of samples evaluated (excluding NaN). Serialized as null when the sample size is unknown "
         "— e.g. a figure imported from a backend that reports statistics without the n behind them. "
         "Distinct from 0, which asserts that nothing was evaluated.",
     )
@@ -322,11 +322,11 @@ class AggregateScoreBase(BaseModel):
         default=None,
         description="Sample standard deviation of the scores (Bessel-corrected, divides by n-1). Estimates "
         "the spread of the process the values were drawn from — the right choice when repeated trials "
-        "sample a stochastic system. None when fewer than two values (undefined, not zero).",
+        "sample a stochastic system. Omitted when fewer than two values (undefined, not zero).",
     )
     sample_variance: float | None = Field(
         default=None,
-        description="Sample variance of the scores (Bessel-corrected, divides by n-1). None when fewer "
+        description="Sample variance of the scores (Bessel-corrected, divides by n-1). Omitted when fewer "
         "than two values.",
     )
 

packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/openapi_utils.py

@@ -8,6 +8,7 @@
 from json import JSONDecodeError
 from typing import Any, Dict, List, Optional
 
+from fastapi import FastAPI
 from pydantic import BaseModel
 from starlette.datastructures import QueryParams
 
@@ -95,26 +96,85 @@ def clear_query_param_schemas() -> None:
     _query_param_schemas.clear()
 
 
+def _strip_null_defaults(schema: Any) -> Any:
+    """Return *schema* without cosmetic ``default: null`` entries."""
+    if isinstance(schema, dict):
+        return {
+            key: _strip_null_defaults(value)
+            for key, value in schema.items()
+            if not (key == "default" and value is None)
+        }
+    if isinstance(schema, list):
+        return [_strip_null_defaults(item) for item in schema]
+    return schema
+
+
+def _promote_schema_defs(schema_name: str, schema: Dict[str, Any], components: Dict[str, Any]) -> None:
+    """Move a schema's local ``$defs`` into ``components.schemas``.
+
+    Query filter schemas are emitted with refs like
+    ``#/components/schemas/DatetimeFilter``. Those refs only resolve if the
+    matching nested definitions are promoted to global components.
+    """
+    local_defs = schema.pop("$defs", None)
+    if not local_defs:
+        return
+
+    for name, local_def in local_defs.items():
+        existing = components.get(name)
+        if existing is None:
+            components[name] = local_def
+            continue
+        if existing == local_def or _strip_null_defaults(existing) == _strip_null_defaults(local_def):
+            continue
+        raise ValueError(
+            f"Schema '{name}' from query parameter schema '{schema_name}' conflicts with existing component"
+        )
+
+
 def register_query_param_schemas(spec: Dict[str, Any]) -> Dict[str, Any]:
     """Inject filter/search classes registered via ``generate_openapi_extra_params``
     into ``components.schemas`` of ``spec`` if not already present.
 
-    The raw ``model_json_schema`` output is dropped in as-is, including any nested
-    ``$defs``. The downstream ``hoist_nested_defs`` pass is the single consolidator
-    that hoists these to top-level components with structural-equality dedup, so
-    duplicates emitted by other sites (e.g. the jobs factory's inline
-    ``openapi_extra``) don't diverge from the copy we inject here.
+    Nested ``$defs`` are promoted because these schemas use component-level refs
+    (``#/components/schemas/{model}``) for nested filter models. Without promotion,
+    live ``/openapi.json`` output can contain dangling refs until the offline
+    postprocessing pipeline runs.
     """
     if not _query_param_schemas:
         return spec
     components = spec.setdefault("components", {}).setdefault("schemas", {})
     for name, cls in _query_param_schemas.items():
         if name in components:
+            existing = components[name]
+            if isinstance(existing, dict):
+                _promote_schema_defs(name, existing, components)
             continue
-        components[name] = cls.model_json_schema(ref_template="#/components/schemas/{model}")
+        schema = cls.model_json_schema(ref_template="#/components/schemas/{model}")
+        _promote_schema_defs(name, schema, components)
+        components[name] = schema
     return spec
 
 
+def install_query_param_schema_openapi_hook(app: FastAPI) -> None:
+    """Wrap ``app.openapi`` so live specs include registered query-param schemas."""
+    default_openapi = app.openapi
+
+    def custom_openapi() -> dict[str, Any]:
+        if app.openapi_schema:
+            return app.openapi_schema
+        openapi_schema = default_openapi()
+        try:
+            openapi_schema = register_query_param_schemas(openapi_schema)
+        except Exception:
+            app.openapi_schema = None
+            raise
+        app.openapi_schema = openapi_schema
+        return app.openapi_schema
+
+    app.openapi = custom_openapi  # type: ignore[method-assign]
+
+
 def parse_deep_object(name: str, params: QueryParams) -> Dict:
     """ "Helper function to parse 'deepObject'-like query parameters."""
     result = {}

packages/nmp_common/src/nmp/common/api/utils.py

@@ -16,6 +16,9 @@
 from nemo_platform_plugin.jobs.openapi_utils import (
     generate_openapi_extra_params as generate_openapi_extra_params,  # noqa: F401
 )
+from nemo_platform_plugin.jobs.openapi_utils import (
+    install_query_param_schema_openapi_hook as install_query_param_schema_openapi_hook,  # noqa: F401
+)
 from nemo_platform_plugin.jobs.openapi_utils import parse_deep_object as parse_deep_object  # noqa: F401
 from nemo_platform_plugin.jobs.openapi_utils import (
     register_query_param_schemas as register_query_param_schemas,  # noqa: F401

packages/nmp_common/tests/api/test_query_param_schemas.py

@@ -12,22 +12,39 @@
 
 from typing import Optional
 
+import nemo_platform_plugin.jobs.openapi_utils as job_openapi_utils
 import pytest
 from fastapi import FastAPI, Query, Request
-from fastapi.openapi.utils import get_openapi
 from fastapi.testclient import TestClient
 from nmp.common.api.utils import (
     clear_query_param_schemas,
     generate_openapi_extra_params,
+    install_query_param_schema_openapi_hook,
     register_query_param_schemas,
 )
-from pydantic import BaseModel
+from pydantic import BaseModel, create_model
 
 
 class _DummyFilter(BaseModel):
     type: Optional[str] = None
 
 
+class DummyDatetimeFilter(BaseModel):
+    gte: Optional[str] = None
+    lte: Optional[str] = None
+
+
+class DummyStringFilter(BaseModel):
+    eq: Optional[str] = None
+    like: Optional[str] = None
+
+
+class DummyJobsListFilter(BaseModel):
+    created_at: Optional[DummyDatetimeFilter] = None
+    name: Optional[DummyStringFilter | str] = None
+    updated_at: Optional[DummyDatetimeFilter] = None
+
+
 @pytest.fixture(autouse=True)
 def _reset_registry():
     """The registry is module-level global state; reset around each test."""
@@ -59,6 +76,28 @@ def test_register_preserves_existing_schemas():
     assert "_DummyFilter" in spec["components"]["schemas"]
 
 
+def test_register_promotes_nested_filter_defs():
+    """Nested filter refs should resolve without depending on offline postprocessing."""
+    generate_openapi_extra_params(filter_schema=DummyJobsListFilter)
+
+    spec = {"components": {"schemas": {}}}
+    spec = register_query_param_schemas(spec)
+    schemas = spec["components"]["schemas"]
+
+    assert "$defs" not in schemas["DummyJobsListFilter"]
+    assert "DummyDatetimeFilter" in schemas
+    assert "DummyStringFilter" in schemas
+    assert schemas["DummyJobsListFilter"]["properties"]["created_at"]["anyOf"][0]["$ref"] == (
+        "#/components/schemas/DummyDatetimeFilter"
+    )
+    assert schemas["DummyJobsListFilter"]["properties"]["name"]["anyOf"][0]["$ref"] == (
+        "#/components/schemas/DummyStringFilter"
+    )
+    assert schemas["DummyJobsListFilter"]["properties"]["updated_at"]["anyOf"][0]["$ref"] == (
+        "#/components/schemas/DummyDatetimeFilter"
+    )
+
+
 def test_clear_resets_registry_between_services():
     generate_openapi_extra_params(filter_schema=_DummyFilter)
     clear_query_param_schemas()
@@ -81,18 +120,46 @@ def test_custom_openapi_hook_resolves_filter_ref():
     async def list_items(request: Request, page: int = Query(default=1)):
         return {"data": []}
 
-    def custom_openapi():
-        if app.openapi_schema:
-            return app.openapi_schema
-        spec = get_openapi(title="t", version="0", routes=app.routes)
-        spec = register_query_param_schemas(spec)
-        app.openapi_schema = spec
-        return spec
-
-    app.openapi = custom_openapi  # type: ignore[method-assign]
+    install_query_param_schema_openapi_hook(app)
 
     spec = TestClient(app).get("/openapi.json").json()
 
     assert "_DummyFilter" in spec["components"]["schemas"]
     param = next(p for p in spec["paths"]["/items"]["get"]["parameters"] if p["name"] == "filter")
     assert param["schema"]["$ref"] == "#/components/schemas/_DummyFilter"
+
+
+def test_custom_openapi_hook_retries_registration_after_component_conflict(monkeypatch):
+    ExistingNested = create_model("ConflictNested", count=(int, ...), __module__="existing_mod")
+    FilterNested = create_model("ConflictNested", value=(str | None, None), __module__="filter_mod")
+
+    class ConflictFilter(BaseModel):
+        created_at: FilterNested | None = None
+
+    app = FastAPI()
+
+    @app.get(
+        "/items",
+        response_model=ExistingNested,
+        openapi_extra=generate_openapi_extra_params(filter_schema=ConflictFilter),
+    )
+    async def list_items() -> dict[str, int]:
+        return {"count": 1}
+
+    attempts = 0
+    original_register = job_openapi_utils.register_query_param_schemas
+
+    def counting_register(spec):
+        nonlocal attempts
+        attempts += 1
+        return original_register(spec)
+
+    monkeypatch.setattr(job_openapi_utils, "register_query_param_schemas", counting_register)
+    install_query_param_schema_openapi_hook(app)
+
+    with pytest.raises(ValueError, match="conflicts with existing component"):
+        app.openapi()
+    with pytest.raises(ValueError, match="conflicts with existing component"):
+        app.openapi()
+
+    assert attempts == 2

packages/nmp_platform_runner/src/nmp/platform_runner/server.py

@@ -19,6 +19,7 @@
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
+from nmp.common.api.utils import install_query_param_schema_openapi_hook
 from nmp.common.auth import AuthorizationMiddleware
 from nmp.common.config import get_auth_config, get_platform_config
 from nmp.common.http_clients import close_shared_http_clients
@@ -275,6 +276,7 @@ async def root_handler() -> Response:
         if service_instance._service_config is not None:
             app.state.service_configs[type(service_instance._service_config)] = service_instance._service_config
 
+    install_query_param_schema_openapi_hook(app)
     app.add_exception_handler(Exception, platform_global_exception_handler)
     return app
 

packages/nmp_platform_runner/tests/test_server.py

@@ -11,14 +11,16 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
-from fastapi import FastAPI
+from fastapi import APIRouter, FastAPI
 from fastapi.testclient import TestClient
+from nemo_platform_plugin.jobs.openapi_utils import clear_query_param_schemas, generate_openapi_extra_params
 from nmp.common.config import AuthConfig, Configuration
 from nmp.common.config.base import OIDCConfig
-from nmp.common.service import Service
+from nmp.common.service import RouterConfig, Service
 from nmp.platform_runner import config as runner_config
 from nmp.platform_runner import server
 from nmp.platform_runner.health import ReadinessCheck, create_platform_health_router
+from pydantic import BaseModel
 
 _RUN_ENV_KEYS = (
     "NMP_CONFIG_FILE_PATH",
@@ -60,6 +62,28 @@ def get_routers(self):
         return []
 
 
+class _DateFilter(BaseModel):
+    gte: str | None = None
+
+
+class _ListFilter(BaseModel):
+    created_at: _DateFilter | None = None
+
+
+class QueryParamSchemaService(Service):
+    def __init__(self):
+        super().__init__(name="query-service", module_name="test.query_service")
+
+    def get_routers(self):
+        router = APIRouter()
+
+        @router.get("/items", openapi_extra=generate_openapi_extra_params(filter_schema=_ListFilter))
+        async def list_items():
+            return {"data": []}
+
+        return [RouterConfig(router, tag="Query", description="Query endpoints")]
+
+
 async def _ready() -> bool:
     return True
 
@@ -174,6 +198,27 @@ def test_create_app_marks_mounted_services_as_local(monkeypatch):
     assert platform_cfg.services == "agents"
 
 
+def test_create_app_openapi_registers_rebased_query_param_schemas(monkeypatch):
+    _patch_platform_app_config(monkeypatch, seed_on_startup=False)
+    clear_query_param_schemas()
+    try:
+        app = server.create_app(services=[QueryParamSchemaService()])
+        spec = app.openapi()
+        schemas = spec["components"]["schemas"]
+        filter_param = next(
+            param
+            for param in spec["paths"]["/apis/query-service/items"]["get"]["parameters"]
+            if param["name"] == "filter"
+        )
+
+        assert filter_param["schema"] == {"$ref": "#/components/schemas/_ListFilter"}
+        assert "_ListFilter" in schemas
+        assert "_DateFilter" in schemas
+        assert "$defs" not in schemas["_ListFilter"]
+    finally:
+        clear_query_param_schemas()
+
+
 def test_create_app_mounted_services_drive_sdk_local_routing_without_services_env(monkeypatch):
     monkeypatch.delenv("NMP_SERVICES", raising=False)
     monkeypatch.setenv("NMP_BASE_URL", "https://nemo-gateway:8080")