From 7c01f7f5ea4a811b0c537a69f4d77578495fd6d0 Mon Sep 17 00:00:00 2001 From: Ryan S <267728323+ironcommit@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:29:13 -0700 Subject: [PATCH] fix(eval): update openapi spec Signed-off-by: Ryan S <267728323+ironcommit@users.noreply.github.com> --- .github/actions/changes/action.yaml | 9 ++ .github/workflows/ci.yaml | 4 + .../src/nemo_evaluator_sdk/values/results.py | 6 +- .../jobs/openapi_utils.py | 72 +++++++++++++-- .../nmp_common/src/nmp/common/api/utils.py | 3 + .../tests/api/test_query_param_schemas.py | 89 ++++++++++++++++--- .../src/nmp/platform_runner/server.py | 2 + .../nmp_platform_runner/tests/test_server.py | 49 +++++++++- plugins/nemo-evaluator/openapi/openapi.yaml | 36 ++++---- script/openapi_helper/plugin_loader.py | 2 + .../beta/evaluator/values/results.py | 6 +- tests/unit/test_plugin_openapi_loader.py | 54 +++++++++++ tools/lint/lint-openapi.sh | 26 ++++-- .../src/api/evaluation/agent-evaluations.ts | 2 +- .../EvalComparisonTable/utils.test.ts | 8 ++ .../dataViews/EvalComparisonTable/utils.ts | 10 ++- .../evaluation/EvalAggregateScoresTable.tsx | 10 ++- 17 files changed, 334 insertions(+), 54 deletions(-) create mode 100644 tests/unit/test_plugin_openapi_loader.py diff --git a/.github/actions/changes/action.yaml b/.github/actions/changes/action.yaml index 00db39d1dd..89ebd57552 100644 --- a/.github/actions/changes/action.yaml +++ b/.github/actions/changes/action.yaml @@ -73,6 +73,15 @@ runs: openapi: - 'openapi/**' - 'plugins/*/openapi/**' + - 'plugins/*/src/**' + - 'plugins/*/pyproject.toml' + - 'packages/nmp_common/src/nmp_common/api/**' + - 'packages/nmp_common/src/nmp_common/datamodel/**' + - 'packages/nemo_platform_plugin/src/**' + - 'packages/nemo_evaluator_sdk/src/**' + - 'script/generate_openapi_spec.py' + - 'script/generate-openapi-spec.sh' + - 'script/openapi_helper/**' tests: - 'tests/**' - 'pytest.ini' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26d36950b7..d0f9939ab5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -787,6 +787,8 @@ jobs: enable-cache: true python-version: "3.12" cache-dependency-glob: uv.lock + - name: Check uv lockfile + run: uv lock --check - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '22' @@ -1628,6 +1630,8 @@ jobs: run: pnpm --filter @nemo/sdk gen:all-force - name: Typecheck SDK generated output run: pnpm --filter @nemo/sdk typecheck + - name: Typecheck web packages against regenerated SDK + run: pnpm run --recursive --parallel --if-present typecheck web-studio-deps: name: Web studio deps check diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py index e1d0a53837..a5d0a385f3 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py @@ -295,7 +295,7 @@ class AggregateScoreBase(BaseModel): name: str = Field(description="Name of the score.") count: int | None = Field( default=None, - description="Number of samples evaluated (excluding NaN). None when the sample size is unknown " + description="Number of samples evaluated (excluding NaN). Serialized as null when the sample size is unknown " "— e.g. a figure imported from a backend that reports statistics without the n behind them. " "Distinct from 0, which asserts that nothing was evaluated.", ) @@ -322,11 +322,11 @@ class AggregateScoreBase(BaseModel): default=None, description="Sample standard deviation of the scores (Bessel-corrected, divides by n-1). Estimates " "the spread of the process the values were drawn from — the right choice when repeated trials " - "sample a stochastic system. None when fewer than two values (undefined, not zero).", + "sample a stochastic system. Omitted when fewer than two values (undefined, not zero).", ) sample_variance: float | None = Field( default=None, - description="Sample variance of the scores (Bessel-corrected, divides by n-1). None when fewer " + description="Sample variance of the scores (Bessel-corrected, divides by n-1). Omitted when fewer " "than two values.", ) diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/openapi_utils.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/openapi_utils.py index 7f7a96e891..281b6ba7a3 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/openapi_utils.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/openapi_utils.py @@ -8,6 +8,7 @@ from json import JSONDecodeError from typing import Any, Dict, List, Optional +from fastapi import FastAPI from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -95,26 +96,85 @@ def clear_query_param_schemas() -> None: _query_param_schemas.clear() +def _strip_null_defaults(schema: Any) -> Any: + """Return *schema* without cosmetic ``default: null`` entries.""" + if isinstance(schema, dict): + return { + key: _strip_null_defaults(value) + for key, value in schema.items() + if not (key == "default" and value is None) + } + if isinstance(schema, list): + return [_strip_null_defaults(item) for item in schema] + return schema + + +def _promote_schema_defs(schema_name: str, schema: Dict[str, Any], components: Dict[str, Any]) -> None: + """Move a schema's local ``$defs`` into ``components.schemas``. + + Query filter schemas are emitted with refs like + ``#/components/schemas/DatetimeFilter``. Those refs only resolve if the + matching nested definitions are promoted to global components. + """ + local_defs = schema.pop("$defs", None) + if not local_defs: + return + + for name, local_def in local_defs.items(): + existing = components.get(name) + if existing is None: + components[name] = local_def + continue + if existing == local_def or _strip_null_defaults(existing) == _strip_null_defaults(local_def): + continue + raise ValueError( + f"Schema '{name}' from query parameter schema '{schema_name}' conflicts with existing component" + ) + + def register_query_param_schemas(spec: Dict[str, Any]) -> Dict[str, Any]: """Inject filter/search classes registered via ``generate_openapi_extra_params`` into ``components.schemas`` of ``spec`` if not already present. - The raw ``model_json_schema`` output is dropped in as-is, including any nested - ``$defs``. The downstream ``hoist_nested_defs`` pass is the single consolidator - that hoists these to top-level components with structural-equality dedup, so - duplicates emitted by other sites (e.g. the jobs factory's inline - ``openapi_extra``) don't diverge from the copy we inject here. + Nested ``$defs`` are promoted because these schemas use component-level refs + (``#/components/schemas/{model}``) for nested filter models. Without promotion, + live ``/openapi.json`` output can contain dangling refs until the offline + postprocessing pipeline runs. """ if not _query_param_schemas: return spec components = spec.setdefault("components", {}).setdefault("schemas", {}) for name, cls in _query_param_schemas.items(): if name in components: + existing = components[name] + if isinstance(existing, dict): + _promote_schema_defs(name, existing, components) continue - components[name] = cls.model_json_schema(ref_template="#/components/schemas/{model}") + schema = cls.model_json_schema(ref_template="#/components/schemas/{model}") + _promote_schema_defs(name, schema, components) + components[name] = schema return spec +def install_query_param_schema_openapi_hook(app: FastAPI) -> None: + """Wrap ``app.openapi`` so live specs include registered query-param schemas.""" + default_openapi = app.openapi + + def custom_openapi() -> dict[str, Any]: + if app.openapi_schema: + return app.openapi_schema + openapi_schema = default_openapi() + try: + openapi_schema = register_query_param_schemas(openapi_schema) + except Exception: + app.openapi_schema = None + raise + app.openapi_schema = openapi_schema + return app.openapi_schema + + app.openapi = custom_openapi # type: ignore[method-assign] + + def parse_deep_object(name: str, params: QueryParams) -> Dict: """ "Helper function to parse 'deepObject'-like query parameters.""" result = {} diff --git a/packages/nmp_common/src/nmp/common/api/utils.py b/packages/nmp_common/src/nmp/common/api/utils.py index 52616be4eb..565a90e636 100644 --- a/packages/nmp_common/src/nmp/common/api/utils.py +++ b/packages/nmp_common/src/nmp/common/api/utils.py @@ -16,6 +16,9 @@ from nemo_platform_plugin.jobs.openapi_utils import ( generate_openapi_extra_params as generate_openapi_extra_params, # noqa: F401 ) +from nemo_platform_plugin.jobs.openapi_utils import ( + install_query_param_schema_openapi_hook as install_query_param_schema_openapi_hook, # noqa: F401 +) from nemo_platform_plugin.jobs.openapi_utils import parse_deep_object as parse_deep_object # noqa: F401 from nemo_platform_plugin.jobs.openapi_utils import ( register_query_param_schemas as register_query_param_schemas, # noqa: F401 diff --git a/packages/nmp_common/tests/api/test_query_param_schemas.py b/packages/nmp_common/tests/api/test_query_param_schemas.py index 5b9236e8db..06cbf89c14 100644 --- a/packages/nmp_common/tests/api/test_query_param_schemas.py +++ b/packages/nmp_common/tests/api/test_query_param_schemas.py @@ -12,22 +12,39 @@ from typing import Optional +import nemo_platform_plugin.jobs.openapi_utils as job_openapi_utils import pytest from fastapi import FastAPI, Query, Request -from fastapi.openapi.utils import get_openapi from fastapi.testclient import TestClient from nmp.common.api.utils import ( clear_query_param_schemas, generate_openapi_extra_params, + install_query_param_schema_openapi_hook, register_query_param_schemas, ) -from pydantic import BaseModel +from pydantic import BaseModel, create_model class _DummyFilter(BaseModel): type: Optional[str] = None +class DummyDatetimeFilter(BaseModel): + gte: Optional[str] = None + lte: Optional[str] = None + + +class DummyStringFilter(BaseModel): + eq: Optional[str] = None + like: Optional[str] = None + + +class DummyJobsListFilter(BaseModel): + created_at: Optional[DummyDatetimeFilter] = None + name: Optional[DummyStringFilter | str] = None + updated_at: Optional[DummyDatetimeFilter] = None + + @pytest.fixture(autouse=True) def _reset_registry(): """The registry is module-level global state; reset around each test.""" @@ -59,6 +76,28 @@ def test_register_preserves_existing_schemas(): assert "_DummyFilter" in spec["components"]["schemas"] +def test_register_promotes_nested_filter_defs(): + """Nested filter refs should resolve without depending on offline postprocessing.""" + generate_openapi_extra_params(filter_schema=DummyJobsListFilter) + + spec = {"components": {"schemas": {}}} + spec = register_query_param_schemas(spec) + schemas = spec["components"]["schemas"] + + assert "$defs" not in schemas["DummyJobsListFilter"] + assert "DummyDatetimeFilter" in schemas + assert "DummyStringFilter" in schemas + assert schemas["DummyJobsListFilter"]["properties"]["created_at"]["anyOf"][0]["$ref"] == ( + "#/components/schemas/DummyDatetimeFilter" + ) + assert schemas["DummyJobsListFilter"]["properties"]["name"]["anyOf"][0]["$ref"] == ( + "#/components/schemas/DummyStringFilter" + ) + assert schemas["DummyJobsListFilter"]["properties"]["updated_at"]["anyOf"][0]["$ref"] == ( + "#/components/schemas/DummyDatetimeFilter" + ) + + def test_clear_resets_registry_between_services(): generate_openapi_extra_params(filter_schema=_DummyFilter) clear_query_param_schemas() @@ -81,18 +120,46 @@ def test_custom_openapi_hook_resolves_filter_ref(): async def list_items(request: Request, page: int = Query(default=1)): return {"data": []} - def custom_openapi(): - if app.openapi_schema: - return app.openapi_schema - spec = get_openapi(title="t", version="0", routes=app.routes) - spec = register_query_param_schemas(spec) - app.openapi_schema = spec - return spec - - app.openapi = custom_openapi # type: ignore[method-assign] + install_query_param_schema_openapi_hook(app) spec = TestClient(app).get("/openapi.json").json() assert "_DummyFilter" in spec["components"]["schemas"] param = next(p for p in spec["paths"]["/items"]["get"]["parameters"] if p["name"] == "filter") assert param["schema"]["$ref"] == "#/components/schemas/_DummyFilter" + + +def test_custom_openapi_hook_retries_registration_after_component_conflict(monkeypatch): + ExistingNested = create_model("ConflictNested", count=(int, ...), __module__="existing_mod") + FilterNested = create_model("ConflictNested", value=(str | None, None), __module__="filter_mod") + + class ConflictFilter(BaseModel): + created_at: FilterNested | None = None + + app = FastAPI() + + @app.get( + "/items", + response_model=ExistingNested, + openapi_extra=generate_openapi_extra_params(filter_schema=ConflictFilter), + ) + async def list_items() -> dict[str, int]: + return {"count": 1} + + attempts = 0 + original_register = job_openapi_utils.register_query_param_schemas + + def counting_register(spec): + nonlocal attempts + attempts += 1 + return original_register(spec) + + monkeypatch.setattr(job_openapi_utils, "register_query_param_schemas", counting_register) + install_query_param_schema_openapi_hook(app) + + with pytest.raises(ValueError, match="conflicts with existing component"): + app.openapi() + with pytest.raises(ValueError, match="conflicts with existing component"): + app.openapi() + + assert attempts == 2 diff --git a/packages/nmp_platform_runner/src/nmp/platform_runner/server.py b/packages/nmp_platform_runner/src/nmp/platform_runner/server.py index eef0e55d2c..01f38787b0 100644 --- a/packages/nmp_platform_runner/src/nmp/platform_runner/server.py +++ b/packages/nmp_platform_runner/src/nmp/platform_runner/server.py @@ -19,6 +19,7 @@ from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +from nmp.common.api.utils import install_query_param_schema_openapi_hook from nmp.common.auth import AuthorizationMiddleware from nmp.common.config import get_auth_config, get_platform_config from nmp.common.http_clients import close_shared_http_clients @@ -275,6 +276,7 @@ async def root_handler() -> Response: if service_instance._service_config is not None: app.state.service_configs[type(service_instance._service_config)] = service_instance._service_config + install_query_param_schema_openapi_hook(app) app.add_exception_handler(Exception, platform_global_exception_handler) return app diff --git a/packages/nmp_platform_runner/tests/test_server.py b/packages/nmp_platform_runner/tests/test_server.py index 2087de75f1..e368b64e51 100644 --- a/packages/nmp_platform_runner/tests/test_server.py +++ b/packages/nmp_platform_runner/tests/test_server.py @@ -11,14 +11,16 @@ from unittest.mock import MagicMock, patch import pytest -from fastapi import FastAPI +from fastapi import APIRouter, FastAPI from fastapi.testclient import TestClient +from nemo_platform_plugin.jobs.openapi_utils import clear_query_param_schemas, generate_openapi_extra_params from nmp.common.config import AuthConfig, Configuration from nmp.common.config.base import OIDCConfig -from nmp.common.service import Service +from nmp.common.service import RouterConfig, Service from nmp.platform_runner import config as runner_config from nmp.platform_runner import server from nmp.platform_runner.health import ReadinessCheck, create_platform_health_router +from pydantic import BaseModel _RUN_ENV_KEYS = ( "NMP_CONFIG_FILE_PATH", @@ -60,6 +62,28 @@ def get_routers(self): return [] +class _DateFilter(BaseModel): + gte: str | None = None + + +class _ListFilter(BaseModel): + created_at: _DateFilter | None = None + + +class QueryParamSchemaService(Service): + def __init__(self): + super().__init__(name="query-service", module_name="test.query_service") + + def get_routers(self): + router = APIRouter() + + @router.get("/items", openapi_extra=generate_openapi_extra_params(filter_schema=_ListFilter)) + async def list_items(): + return {"data": []} + + return [RouterConfig(router, tag="Query", description="Query endpoints")] + + async def _ready() -> bool: return True @@ -174,6 +198,27 @@ def test_create_app_marks_mounted_services_as_local(monkeypatch): assert platform_cfg.services == "agents" +def test_create_app_openapi_registers_rebased_query_param_schemas(monkeypatch): + _patch_platform_app_config(monkeypatch, seed_on_startup=False) + clear_query_param_schemas() + try: + app = server.create_app(services=[QueryParamSchemaService()]) + spec = app.openapi() + schemas = spec["components"]["schemas"] + filter_param = next( + param + for param in spec["paths"]["/apis/query-service/items"]["get"]["parameters"] + if param["name"] == "filter" + ) + + assert filter_param["schema"] == {"$ref": "#/components/schemas/_ListFilter"} + assert "_ListFilter" in schemas + assert "_DateFilter" in schemas + assert "$defs" not in schemas["_ListFilter"] + finally: + clear_query_param_schemas() + + def test_create_app_mounted_services_drive_sdk_local_routing_without_services_env(monkeypatch): monkeypatch.delenv("NMP_SERVICES", raising=False) monkeypatch.setenv("NMP_BASE_URL", "https://nemo-gateway:8080") diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml index 130dd09cc2..366017a676 100644 --- a/plugins/nemo-evaluator/openapi/openapi.yaml +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -2676,10 +2676,10 @@ components: description: Name of the score. count: title: Count - description: "Number of samples evaluated (excluding NaN). None when the\ - \ sample size is unknown \u2014 e.g. a figure imported from a backend\ - \ that reports statistics without the n behind them. Distinct from 0,\ - \ which asserts that nothing was evaluated." + description: "Number of samples evaluated (excluding NaN). Serialized as\ + \ null when the sample size is unknown \u2014 e.g. a figure imported from\ + \ a backend that reports statistics without the n behind them. Distinct\ + \ from 0, which asserts that nothing was evaluated." type: integer nan_count: type: integer @@ -2722,12 +2722,12 @@ components: description: "Sample standard deviation of the scores (Bessel-corrected,\ \ divides by n-1). Estimates the spread of the process the values were\ \ drawn from \u2014 the right choice when repeated trials sample a stochastic\ - \ system. None when fewer than two values (undefined, not zero)." + \ system. Omitted when fewer than two values (undefined, not zero)." type: number sample_variance: title: Sample Variance description: Sample variance of the scores (Bessel-corrected, divides by - n-1). None when fewer than two values. + n-1). Omitted when fewer than two values. type: number score_type: type: string @@ -2759,10 +2759,10 @@ components: description: Name of the score. count: title: Count - description: "Number of samples evaluated (excluding NaN). None when the\ - \ sample size is unknown \u2014 e.g. a figure imported from a backend\ - \ that reports statistics without the n behind them. Distinct from 0,\ - \ which asserts that nothing was evaluated." + description: "Number of samples evaluated (excluding NaN). Serialized as\ + \ null when the sample size is unknown \u2014 e.g. a figure imported from\ + \ a backend that reports statistics without the n behind them. Distinct\ + \ from 0, which asserts that nothing was evaluated." type: integer nan_count: type: integer @@ -2805,12 +2805,12 @@ components: description: "Sample standard deviation of the scores (Bessel-corrected,\ \ divides by n-1). Estimates the spread of the process the values were\ \ drawn from \u2014 the right choice when repeated trials sample a stochastic\ - \ system. None when fewer than two values (undefined, not zero)." + \ system. Omitted when fewer than two values (undefined, not zero)." type: number sample_variance: title: Sample Variance description: Sample variance of the scores (Bessel-corrected, divides by - n-1). None when fewer than two values. + n-1). Omitted when fewer than two values. type: number score_type: type: string @@ -2844,10 +2844,10 @@ components: description: Name of the score. count: title: Count - description: "Number of samples evaluated (excluding NaN). None when the\ - \ sample size is unknown \u2014 e.g. a figure imported from a backend\ - \ that reports statistics without the n behind them. Distinct from 0,\ - \ which asserts that nothing was evaluated." + description: "Number of samples evaluated (excluding NaN). Serialized as\ + \ null when the sample size is unknown \u2014 e.g. a figure imported from\ + \ a backend that reports statistics without the n behind them. Distinct\ + \ from 0, which asserts that nothing was evaluated." type: integer nan_count: type: integer @@ -2890,12 +2890,12 @@ components: description: "Sample standard deviation of the scores (Bessel-corrected,\ \ divides by n-1). Estimates the spread of the process the values were\ \ drawn from \u2014 the right choice when repeated trials sample a stochastic\ - \ system. None when fewer than two values (undefined, not zero)." + \ system. Omitted when fewer than two values (undefined, not zero)." type: number sample_variance: title: Sample Variance description: Sample variance of the scores (Bessel-corrected, divides by - n-1). None when fewer than two values. + n-1). Omitted when fewer than two values. type: number score_type: type: string diff --git a/script/openapi_helper/plugin_loader.py b/script/openapi_helper/plugin_loader.py index d111e81c28..301a244c57 100644 --- a/script/openapi_helper/plugin_loader.py +++ b/script/openapi_helper/plugin_loader.py @@ -19,6 +19,7 @@ from fastapi import FastAPI from nemo_platform_plugin.discovery import discover_services +from nmp.common.api.utils import install_query_param_schema_openapi_hook from nmp.platform_runner.plugin_adapter import NemoServiceAdapter @@ -32,4 +33,5 @@ def build_plugin_app(plugin_name: str) -> FastAPI: parent = FastAPI(title=f"{plugin_name} (plugin)") parent.include_router(sub_app.router, prefix=f"/apis/{plugin_name}") + install_query_param_schema_openapi_hook(parent) return parent diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py index 462d0e0ace..47083eee8f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py @@ -295,7 +295,7 @@ class AggregateScoreBase(BaseModel): name: str = Field(description="Name of the score.") count: int | None = Field( default=None, - description="Number of samples evaluated (excluding NaN). None when the sample size is unknown " + description="Number of samples evaluated (excluding NaN). Serialized as null when the sample size is unknown " "— e.g. a figure imported from a backend that reports statistics without the n behind them. " "Distinct from 0, which asserts that nothing was evaluated.", ) @@ -322,11 +322,11 @@ class AggregateScoreBase(BaseModel): default=None, description="Sample standard deviation of the scores (Bessel-corrected, divides by n-1). Estimates " "the spread of the process the values were drawn from — the right choice when repeated trials " - "sample a stochastic system. None when fewer than two values (undefined, not zero).", + "sample a stochastic system. Omitted when fewer than two values (undefined, not zero).", ) sample_variance: float | None = Field( default=None, - description="Sample variance of the scores (Bessel-corrected, divides by n-1). None when fewer " + description="Sample variance of the scores (Bessel-corrected, divides by n-1). Omitted when fewer " "than two values.", ) diff --git a/tests/unit/test_plugin_openapi_loader.py b/tests/unit/test_plugin_openapi_loader.py new file mode 100644 index 0000000000..30fcaa492a --- /dev/null +++ b/tests/unit/test_plugin_openapi_loader.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from fastapi import APIRouter +from nemo_platform_plugin.jobs.openapi_utils import clear_query_param_schemas, generate_openapi_extra_params +from nemo_platform_plugin.service import NemoService, RouterSpec +from pydantic import BaseModel + +from script.openapi_helper.openapi_tools import validate_refs +from script.openapi_helper.plugin_loader import build_plugin_app + + +class _DateFilter(BaseModel): + gte: str | None = None + + +class _ListFilter(BaseModel): + created_at: _DateFilter | None = None + + +class _PluginService(NemoService): + name = "widgets" + + def get_routers(self) -> list[RouterSpec]: + router = APIRouter() + + @router.get("/items", openapi_extra=generate_openapi_extra_params(filter_schema=_ListFilter)) + async def list_items() -> dict[str, list[object]]: + return {"data": []} + + return [RouterSpec(router=router, prefix="/v1")] + + +def test_build_plugin_app_openapi_registers_rebased_query_param_schemas(monkeypatch): + clear_query_param_schemas() + monkeypatch.setattr( + "script.openapi_helper.plugin_loader.discover_services", + lambda: {"widgets": _PluginService}, + ) + try: + app = build_plugin_app("widgets") + spec = app.openapi() + schemas = spec["components"]["schemas"] + filter_param = next( + param for param in spec["paths"]["/apis/widgets/v1/items"]["get"]["parameters"] if param["name"] == "filter" + ) + + assert filter_param["schema"] == {"$ref": "#/components/schemas/_ListFilter"} + assert "_ListFilter" in schemas + assert "_DateFilter" in schemas + assert "$defs" not in schemas["_ListFilter"] + assert validate_refs(spec) == [] + finally: + clear_query_param_schemas() diff --git a/tools/lint/lint-openapi.sh b/tools/lint/lint-openapi.sh index 8a98ddef31..cfddaf8764 100755 --- a/tools/lint/lint-openapi.sh +++ b/tools/lint/lint-openapi.sh @@ -5,9 +5,25 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="${CI_PROJECT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" cd "${PROJECT_ROOT}" -mkdir -p openapicheck -cp openapi/openapi.yaml openapicheck/openapi.yaml -cp openapi/ga/openapi.yaml openapicheck/openapi.ga.yaml +check_dir="openapicheck" +rm -rf "${check_dir}" +mkdir -p "${check_dir}" + +mapfile -t spec_files < <(git ls-files 'openapi/**/*.yaml' 'openapi/*.yaml' 'plugins/*/openapi/openapi.yaml') +for spec_file in "${spec_files[@]}"; do + mkdir -p "${check_dir}/$(dirname "${spec_file}")" + cp "${spec_file}" "${check_dir}/${spec_file}" +done + script/generate-openapi-spec.sh -diff openapi/openapi.yaml openapicheck/openapi.yaml -diff openapi/ga/openapi.yaml openapicheck/openapi.ga.yaml + +for spec_file in "${spec_files[@]}"; do + diff "${check_dir}/${spec_file}" "${spec_file}" +done + +new_plugin_specs="$(git ls-files --others --exclude-standard 'plugins/*/openapi/openapi.yaml')" +if [[ -n "${new_plugin_specs}" ]]; then + echo "New plugin OpenAPI specs were generated and must be committed:" >&2 + echo "${new_plugin_specs}" >&2 + exit 1 +fi diff --git a/web/packages/studio/src/api/evaluation/agent-evaluations.ts b/web/packages/studio/src/api/evaluation/agent-evaluations.ts index b26fef404f..85e420b625 100644 --- a/web/packages/studio/src/api/evaluation/agent-evaluations.ts +++ b/web/packages/studio/src/api/evaluation/agent-evaluations.ts @@ -23,7 +23,7 @@ import { filesDownloadFile } from '@nemo/sdk/generated/platform/api'; const PAGE_SIZE = 50; -/** Aggregate score — numeric range, rubric category distribution, or a single reported value. */ +/** Aggregate score — numeric range, rubric category distribution, or scalar value. */ export type AgentEvalAggregateScore = | AggregateRangeScore | AggregateRubricScore diff --git a/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.test.ts b/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.test.ts index 0bdaad0f32..b00d6a7c93 100644 --- a/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.test.ts +++ b/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.test.ts @@ -162,4 +162,12 @@ describe('comparison score helpers', () => { ] as AgentEvalResult['scores']['scores']) ).toEqual([{ name: 'rubric', mean: null }]); }); + + it('normalizes scalar agent scores from value', () => { + expect( + comparisonScoresForAgentEval([ + { name: 'pass@1', nan_count: 0, score_type: 'scalar', value: 0.72 }, + ]) + ).toEqual([{ name: 'pass@1', mean: 0.72 }]); + }); }); diff --git a/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.ts b/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.ts index 39843eb2ff..d29a37e80f 100644 --- a/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.ts +++ b/web/packages/studio/src/components/dataViews/EvalComparisonTable/utils.ts @@ -39,10 +39,16 @@ export const comparisonsForEvalConfig = ( ]; }); -/** Normalizes optional agent-evaluation means to the comparison table's explicit null value. */ +const aggregateScoreValue = (score: AgentEvalAggregateScore): number | null => { + const value = score.score_type === 'scalar' ? score.value : score.mean; + return typeof value === 'number' && Number.isFinite(value) ? value : null; +}; + +/** Normalizes optional agent-evaluation values to the comparison table's explicit null value. */ export const comparisonScoresForAgentEval = ( scores: readonly AgentEvalAggregateScore[] -): EvalComparisonScore[] => scores.map(({ name, mean }) => ({ name, mean: mean ?? null })); +): EvalComparisonScore[] => + scores.map((score) => ({ name: score.name, mean: aggregateScoreValue(score) })); /** Selects the aggregate mean for every metric in the model-evaluation result artifact. * Model results use a record keyed by metric name, whereas agent results are already an diff --git a/web/packages/studio/src/components/evaluation/EvalAggregateScoresTable.tsx b/web/packages/studio/src/components/evaluation/EvalAggregateScoresTable.tsx index fcf65583c4..276cbab423 100644 --- a/web/packages/studio/src/components/evaluation/EvalAggregateScoresTable.tsx +++ b/web/packages/studio/src/components/evaluation/EvalAggregateScoresTable.tsx @@ -14,6 +14,7 @@ export interface EvalAggregateScoreRow { nan_count?: number; min?: number | null; max?: number | null; + value?: number | null; score_type?: string; mode_category?: string | null; rubric_distribution?: { label: string; value?: number; count?: number }[]; @@ -40,6 +41,9 @@ const trialsText = (score: EvalAggregateScoreRow): string => { return `${scored}/${scored + (score.nan_count ?? 0)}`; }; +const displayScoreValue = (score: EvalAggregateScoreRow): number | null => + score.score_type === 'scalar' ? (score.value ?? null) : (score.mean ?? null); + export const EvalAggregateScoresTable: FC = ({ scores, emptyMessage = 'No scores recorded for this evaluation.', @@ -68,14 +72,14 @@ export const EvalAggregateScoresTable: FC = ({ ), }), - col.accessor((original) => original.mean ?? null, { + col.accessor(displayScoreValue, { id: 'score', header: 'Score', enableSorting: false, size: 100, cell: ({ row }) => ( - - {formatScore(row.original.mean)} + + {formatScore(displayScoreValue(row.original))} ), }),