From 9cad1974b9346e6cdf5eb96f7856e1396b71a3f9 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Wed, 12 Aug 2026 17:45:13 -0300 Subject: [PATCH 1/3] refactor(evaluator)!: drop the plugin's local execution path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SKILL.md` already told callers not to build on `client.evaluator.run()` because it was being retired. This removes it, along with the executor methods that existed only to serve it: `run_local`, `evaluate_remote`, `evaluate`, and `evaluate_benchmark` on both the sync and async executors. Nothing in production called it. The plugin's remote path is `submit`, and callers wanting local execution use the standalone `nemo_evaluator_sdk.Evaluator` directly, which is what the skill already recommends. Removing it orphans a supporting cast, so that goes too: `sdk/fs_utils.py` entirely, since `local_artifact_path` was only reachable from `local_result_path` and `EvaluatorLocalRunResult` only from tests of the removed methods; and `filter_evaluation_result`/`filter_benchmark_result` from `sdk/utils.py`, which keeps `filter_aggregate_scores` for `job_resources`. Two `create` tests patched `_executor.asyncio.to_thread` to prove creation never bridges through a thread. With local execution gone the module has no asyncio import at all, so they now assert its absence — the property holds by construction rather than by one observed call. The CLI `run` verb is unaffected: it comes from the job framework's `NemoJobScheduler.run_local`, which is a different path from the executor method of the same name. BREAKING CHANGE: `client.evaluator.run()` is removed from both the sync and async plugin resources. Use `submit` for platform evaluation, or `nemo_evaluator_sdk.Evaluator` for local execution. Co-Authored-By: Claude Opus 5 Signed-off-by: Sandy Chapman --- .../src/nemo_evaluator/sdk/_executor.py | 244 ---------- .../src/nemo_evaluator/sdk/fs_utils.py | 54 --- .../src/nemo_evaluator/sdk/resources.py | 124 ------ .../src/nemo_evaluator/sdk/utils.py | 33 -- plugins/nemo-evaluator/tests/test_sdk.py | 421 +----------------- 5 files changed, 6 insertions(+), 870 deletions(-) delete mode 100644 plugins/nemo-evaluator/src/nemo_evaluator/sdk/fs_utils.py diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py index f1afd7624d..4b5062b28a 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py @@ -5,7 +5,6 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import Any, TypeAlias, cast @@ -15,21 +14,18 @@ from nemo_evaluator.jobs.evaluate import EvaluateInputSpec, EvaluateJob, EvaluateSpec, TargetSpec from nemo_evaluator.resolvers import PlatformModelResolver from nemo_evaluator.sdk import http_utils -from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult, local_result_path from nemo_evaluator.sdk.job_resources import ( AsyncEvaluatorJobResource, EvaluatorJob, EvaluatorJobResource, ) from nemo_evaluator.sdk.types import PluginDatasetInput -from nemo_evaluator.sdk.utils import filter_benchmark_result, filter_evaluation_result from nemo_evaluator.shared.metric_bundles.bundles import ( MetricBundle, MetricBundlePackager, MetricBundlePackagerPolicyError, bundle_metric, ) -from nemo_evaluator.shared.metric_bundles.defaults import resolve_default_metric_bundle_packager from nemo_evaluator_sdk.datasets.loader import prepare_dataset_rows from nemo_evaluator_sdk.execution.config import resolve_params from nemo_evaluator_sdk.execution.metric_execution import run_sync @@ -44,10 +40,7 @@ RunConfigOnline, RunConfigOnlineModel, ) -from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult -from nemo_evaluator_sdk.values.results import AggregateFieldName, EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform_plugin.scheduler import NemoJobScheduler _DEFAULT_POLL_INTERVAL_SECONDS = 10.0 _DEFAULT_JOB_TIMEOUT_SECONDS = 3600.0 @@ -247,90 +240,6 @@ def create( ) return job_resource - def run_local(self, *, spec: EvaluateRequestSpec, workspace: str | None = None) -> EvaluatorLocalRunResult: - """Run an evaluator plugin job locally with a sync platform client.""" - resolved_workspace = http_utils.resolve_workspace(self._platform, workspace) - canonical_spec = _resolve_sync_local_spec( - spec, - platform=self._platform, - workspace=resolved_workspace, - ) - payload = NemoJobScheduler().run_local( - EvaluateJob, - canonical_spec.model_dump(mode="json"), - workspace=resolved_workspace, - sdk=self._platform, - ) - - return EvaluatorLocalRunResult.model_validate(payload) - - def evaluate_remote( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - metric_bundle_packager: MetricBundlePackager | None = None, - ) -> EvaluationResult: - """Submit, poll, and download a remote evaluator plugin metric job.""" - normalized_params = resolve_params(params, target) - spec = _build_evaluate_spec( - metrics=metric, - dataset=dataset, - params=normalized_params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - metric_bundle_packager=metric_bundle_packager, - ) - - job = self.create( - spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True) - ) - job.wait_until_done( - poll_interval_seconds=self._poll_interval_seconds, - job_timeout_seconds=self._job_timeout_seconds, - pending_timeout_seconds=self._pending_timeout_seconds, - ) - - return job.get_result(aggregate_fields=aggregate_fields) - - def evaluate( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: - """Evaluate one metric through local plugin job execution.""" - normalized_params = resolve_params(params, target) - spec = _build_evaluate_spec( - metrics=metric, - dataset=dataset, - params=normalized_params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - metric_bundle_packager=resolve_default_metric_bundle_packager( - metric, None, allow_cloudpickle_fallback=True, action="Running" - ), - ) - payload = self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - result_path = local_result_path(payload) - result = EvaluationResult.model_validate_json(result_path.read_text(encoding="utf-8")) - return filter_evaluation_result(result, aggregate_fields) - def submit( self, *, @@ -361,38 +270,6 @@ def submit( return job - def evaluate_benchmark( - self, - *, - metrics: Sequence[Metric], - dataset: PluginDatasetInput, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> BenchmarkEvaluationResult: - """Evaluate multiple metrics through local plugin job execution.""" - normalized_params = resolve_params(params, target) - spec = _build_evaluate_spec( - metrics=metrics, - dataset=dataset, - params=normalized_params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - metric_bundle_packager=resolve_default_metric_bundle_packager( - metrics, None, allow_cloudpickle_fallback=True, action="Running" - ), - ) - payload = self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - result_path = local_result_path(payload) - result = BenchmarkEvaluationResult.model_validate_json(result_path.read_text(encoding="utf-8")) - return filter_benchmark_result(result, aggregate_fields) - class _AsyncEvaluatorPluginExecutor: """Async evaluator plugin executor used by the async SDK resource.""" @@ -449,26 +326,6 @@ async def create( ) return job_resource - async def run_local(self, *, spec: EvaluateRequestSpec, workspace: str | None = None) -> EvaluatorLocalRunResult: - """Run an evaluator plugin job locally without blocking the event loop.""" - resolved_workspace = http_utils.resolve_workspace(self._platform, workspace) - canonical_spec = await _resolve_async_local_spec( - spec, - platform=self._platform, - workspace=resolved_workspace, - ) - scheduler = NemoJobScheduler() - # Leverages programmatic dispatch as described in - # packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/ARCHITECTURE.md#job-entry-point-keys - payload = await asyncio.to_thread( - scheduler.run_local, - EvaluateJob, - canonical_spec.model_dump(mode="json"), - workspace=resolved_workspace, - async_sdk=self._platform, - ) - return EvaluatorLocalRunResult.model_validate(payload) - async def submit( self, *, @@ -499,107 +356,6 @@ async def submit( return job - async def evaluate_remote( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - metric_bundle_packager: MetricBundlePackager | None = None, - ) -> EvaluationResult: - """Submit, poll, and download a remote evaluator plugin metric job.""" - normalized_params = resolve_params(params, target) - spec = _build_evaluate_spec( - metrics=metric, - dataset=dataset, - params=normalized_params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - metric_bundle_packager=metric_bundle_packager, - ) - - job = await self.create( - spec=spec, workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True) - ) - await job.wait_until_done( - poll_interval_seconds=self._poll_interval_seconds, - job_timeout_seconds=self._job_timeout_seconds, - pending_timeout_seconds=self._pending_timeout_seconds, - ) - - return await job.get_result(aggregate_fields=aggregate_fields) - - async def evaluate( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: - """Evaluate one metric through local plugin job execution.""" - normalized_params = resolve_params(params, target) - spec = _build_evaluate_spec( - metrics=metric, - dataset=dataset, - params=normalized_params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - metric_bundle_packager=resolve_default_metric_bundle_packager( - metric, None, allow_cloudpickle_fallback=True, action="Running" - ), - ) - payload = await self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - result_path = local_result_path(payload) - result_text = await asyncio.to_thread(result_path.read_text, encoding="utf-8") - result = EvaluationResult.model_validate_json(result_text) - return filter_evaluation_result(result, aggregate_fields) - - async def evaluate_benchmark( - self, - *, - metrics: Sequence[Metric], - dataset: PluginDatasetInput, - params: RunConfig | RunConfigOnline | RunConfigOnlineModel, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> BenchmarkEvaluationResult: - """Evaluate multiple metrics through local plugin job execution.""" - normalized_params = resolve_params(params, target) - spec = _build_evaluate_spec( - metrics=metrics, - dataset=dataset, - params=normalized_params, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - metric_bundle_packager=resolve_default_metric_bundle_packager( - metrics, None, allow_cloudpickle_fallback=True, action="Running" - ), - ) - payload = await self.run_local( - spec=spec, - workspace=http_utils.resolve_workspace(self._platform, self._workspace, strict=True), - ) - result_path = local_result_path(payload) - result_text = await asyncio.to_thread(result_path.read_text, encoding="utf-8") - result = BenchmarkEvaluationResult.model_validate_json(result_text) - return filter_benchmark_result(result, aggregate_fields) - def bundle_metrics_for_spec( metrics: Metric | Sequence[Metric], *, metric_bundle_packager: MetricBundlePackager diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/fs_utils.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/fs_utils.py deleted file mode 100644 index 1434f2b302..0000000000 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/fs_utils.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Filesystem helpers for evaluator plugin SDK local result artifacts.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Literal, Self -from urllib.parse import unquote, urlparse - -from nemo_evaluator.jobs.evaluate import DEFAULT_FILE_NAME -from nemo_platform_plugin.job_results import ResultRef -from pydantic import BaseModel, ConfigDict, model_validator - - -class EvaluatorLocalRunResult(BaseModel): - """Validated payload returned by local evaluator plugin job execution.""" - - model_config = ConfigDict(extra="allow") - - status: Literal["completed", "error"] - artifact: ResultRef | None = None - - @model_validator(mode="after") - def require_completed_artifact(self) -> Self: - """Require completed local runs to include a saved result artifact.""" - if self.status == "completed" and self.artifact is None: - raise ValueError("completed local evaluator jobs must include an artifact") - return self - - -def local_result_path(payload: EvaluatorLocalRunResult) -> Path: - """Return the JSON result file path for a completed local evaluator run.""" - if payload.status != "completed": - raise RuntimeError(f"local evaluator job finished with status {payload.status!r}") - if payload.artifact is None: - raise TypeError("local evaluator job response must include an artifact object") - artifact_path = local_artifact_path(payload.artifact) - return artifact_path / DEFAULT_FILE_NAME if artifact_path.is_dir() else artifact_path - - -def local_artifact_path(artifact: ResultRef) -> Path: - """Resolve a local evaluator job artifact reference to a filesystem path.""" - artifact_ref = artifact.artifact_url - - parsed = urlparse(artifact_ref) - if parsed.scheme == "file": - if parsed.netloc and parsed.netloc != "localhost": - raise ValueError(f"local evaluator job artifact URL must point to this host: {artifact_ref!r}") - return Path(unquote(parsed.path)) - if parsed.scheme: - raise ValueError(f"local evaluator job artifact URL must use file:// for local execution: {artifact_ref!r}") - return Path(artifact_ref) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py index 9f6e04bc17..60e0190712 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/resources.py @@ -48,12 +48,10 @@ from nemo_evaluator_sdk.metrics.protocol import Metric from nemo_evaluator_sdk.values import ( Agent, - AggregateFieldName, FieldMapping, Model, ModelRef, ) -from nemo_evaluator_sdk.values.results import EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform_plugin.sdk import NemoPluginSDKResources @@ -166,67 +164,6 @@ def submit( ), ) - @overload - def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfig | None = None, - target: None = None, - field_mapping: FieldMapping | None = None, - prompt_template: None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: ... - - @overload - def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfigOnlineModel, - target: Model, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: ... - - @overload - def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfigOnline, - target: Agent, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any], - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: ... - - def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: - """Run one metric through the evaluator plugin executor's local execution path.""" - return self._executor.evaluate( - metric=metric, - dataset=dataset, - params=config, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - aggregate_fields=aggregate_fields, - ) - class AsyncEvaluator: """Async SDK namespace mounted as ``client.evaluator``.""" @@ -273,67 +210,6 @@ async def get_job_resource(self, job_name: str, workspace: str | None = None) -> headers=http_utils.platform_default_headers(self._platform), ) - @overload - async def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfig | None = None, - target: None = None, - field_mapping: FieldMapping | None = None, - prompt_template: None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: ... - - @overload - async def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfigOnlineModel, - target: Model, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: ... - - @overload - async def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfigOnline, - target: Agent, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any], - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: ... - - async def run( - self, - *, - metric: Metric, - dataset: PluginDatasetInput, - config: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, - target: Model | Agent | None = None, - field_mapping: FieldMapping | None = None, - prompt_template: str | dict[str, Any] | None = None, - aggregate_fields: tuple[AggregateFieldName, ...] | None = None, - ) -> EvaluationResult: - """Run one metric through the evaluator plugin executor's local execution path.""" - return await self._executor.evaluate( - metric=metric, - dataset=dataset, - params=config, - target=target, - field_mapping=field_mapping, - prompt_template=prompt_template, - aggregate_fields=aggregate_fields, - ) - @overload async def submit( self, diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/utils.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/utils.py index ae42659f32..7dbf21950d 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/utils.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/utils.py @@ -5,11 +5,9 @@ from __future__ import annotations -from nemo_evaluator_sdk.values.multi_metric_results import BenchmarkEvaluationResult from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, AggregateFieldName, - EvaluationResult, ) @@ -22,34 +20,3 @@ def filter_aggregate_scores( return aggregate_scores fields = frozenset(aggregate_fields) return AggregatedMetricResult(scores=[score.with_fields(fields) for score in aggregate_scores.scores]) - - -def filter_evaluation_result( - result: EvaluationResult, - aggregate_fields: tuple[AggregateFieldName, ...] | None, -) -> EvaluationResult: - """Apply result-only aggregate projection to one evaluation result.""" - if not aggregate_fields: - return result - return result.model_copy( - update={"aggregate_scores": filter_aggregate_scores(result.aggregate_scores, aggregate_fields)} - ) - - -def filter_benchmark_result( - result: BenchmarkEvaluationResult, - aggregate_fields: tuple[AggregateFieldName, ...] | None, -) -> BenchmarkEvaluationResult: - """Apply result-only aggregate projection to a benchmark result.""" - if not aggregate_fields: - return result - per_metric = { - metric_key: filter_evaluation_result(metric_result, aggregate_fields) - for metric_key, metric_result in result.per_metric.items() - } - return result.model_copy( - update={ - "aggregate_scores": filter_aggregate_scores(result.aggregate_scores, aggregate_fields), - "per_metric": per_metric, - } - ) diff --git a/plugins/nemo-evaluator/tests/test_sdk.py b/plugins/nemo-evaluator/tests/test_sdk.py index 828b88a3d4..587615848a 100644 --- a/plugins/nemo-evaluator/tests/test_sdk.py +++ b/plugins/nemo-evaluator/tests/test_sdk.py @@ -5,7 +5,6 @@ from __future__ import annotations -from pathlib import Path from typing import Any, cast from unittest.mock import AsyncMock, MagicMock @@ -13,8 +12,9 @@ import pytest from nemo_evaluator.api.schemas import MetricInline from nemo_evaluator.filesets import FilesetRef -from nemo_evaluator.jobs.evaluate import EvaluateInputSpec, EvaluateJob, EvaluateSpec +from nemo_evaluator.jobs.evaluate import EvaluateInputSpec, EvaluateSpec from nemo_evaluator.metric_refs import MetricRefOrInline +from nemo_evaluator.sdk import _executor as executor_module from nemo_evaluator.sdk import http_utils from nemo_evaluator.sdk._executor import ( MetricBundlePackagerPolicyError, @@ -24,7 +24,6 @@ bundle_metrics_for_spec, create_job_payload, ) -from nemo_evaluator.sdk.fs_utils import EvaluatorLocalRunResult from nemo_evaluator.sdk.job_resources import AsyncEvaluatorJobResource, EvaluatorJobResource from nemo_evaluator.sdk.resources import AsyncEvaluator, Evaluator from nemo_evaluator.shared.metric_bundles.bundles import ( @@ -37,7 +36,6 @@ from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values import FieldMapping, Model, ModelRef, RunConfig, RunConfigOnline, RunConfigOnlineModel -from nemo_evaluator_sdk.values.results import AggregatedMetricResult, EvaluationResult from nemo_platform import AsyncNeMoPlatform, NeMoPlatform from nemo_platform_plugin.jobs.schemas import PlatformJobStatus from pydantic import ValidationError @@ -83,17 +81,6 @@ def _metric_type(metric: MetricRefOrInline) -> str: return metric.metric_type -def _local_run_result(tmp_path: Path, result: EvaluationResult) -> EvaluatorLocalRunResult: - result_path = tmp_path / "evaluation-results.json" - result_path.write_text(result.model_dump_json(), encoding="utf-8") - return EvaluatorLocalRunResult.model_validate( - { - "status": "completed", - "artifact": {"name": "evaluation-results", "artifact_url": f"file://{result_path}"}, - } - ) - - class _RecordingMetricBundlePackager(MetricBundlePackager): """Test packager that records all runtime metrics selected for packaging.""" @@ -241,29 +228,6 @@ def test_build_evaluate_spec_requires_metric_bundle_packager() -> None: ) -def test_local_run_allows_cloudpickle_fallback_for_custom_metric(mocker: MockerFixture) -> None: - """Local run() executes in the caller's process, so custom metrics fall back to cloudpickle. - - The fallback is enabled only for local execution; remote submit/create still require an - explicit cloudpickle opt-in (covered separately). - """ - import nemo_evaluator.sdk._executor as executor_module - - spy = mocker.spy(executor_module, "resolve_default_metric_bundle_packager") - resource = Evaluator(cast(NeMoPlatform, _SyncPlatform())) - # Short-circuit after packaging so we don't drive the local job runtime. - mocker.patch.object(resource._executor, "run_local", side_effect=RuntimeError("stop after packaging")) - - with pytest.raises(RuntimeError, match="stop after packaging"): - resource.run( - metric=cast(Metric, _CustomRuntimeMetric()), - dataset=[{"expected": "a", "output": "a"}], - ) - - # No MetricBundlePackagerPolicyError: the custom metric was bundled (via cloudpickle) and reached execution. - assert spy.call_args.kwargs["allow_cloudpickle_fallback"] is True - - def test_build_evaluate_spec_includes_target_and_prompt_template() -> None: """Online evaluator specs should preserve model targets and prompt templates.""" model = Model(url="https://model.test/v1", name="model-a") @@ -408,13 +372,14 @@ def test_sync_executor_create_does_not_use_asyncio_thread_bridge(mocker: MockerF request=httpx.Request("POST", "http://test:8000/apis/evaluator/v2/workspaces/ws/evaluate/jobs"), json={"name": "job-123", "status": "created", "spec": _EXACT_MATCH_SPEC}, ) - to_thread = mocker.patch("nemo_evaluator.sdk._executor.asyncio.to_thread", new=AsyncMock(), create=True) executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) job = executor.create(spec=_EXACT_MATCH_EVALUATE_INPUT_SPEC, workspace="ws") assert isinstance(job, EvaluatorJobResource) - to_thread.assert_not_called() + # Stronger than patching `to_thread` and asserting it went uncalled: with local execution gone + # the module has no asyncio import left, so there is no bridge available to reach for. + assert not hasattr(executor_module, "asyncio") platform._client.post.assert_called_once() @@ -535,32 +500,6 @@ def test_sync_resource_url_encodes_reserved_chars_in_job_name() -> None: ) -def test_sync_executor_runs_evaluator_job_locally(mocker: MockerFixture) -> None: - platform = _SyncPlatform() - scheduler = mocker.Mock() - expected = {"status": "completed", "artifact": {"name": "evaluation-results", "artifact_url": "file:///results"}} - scheduler.run_local.return_value = expected - scheduler_cls = mocker.patch("nemo_evaluator.sdk._executor.NemoJobScheduler", return_value=scheduler, create=True) - to_thread = mocker.patch("nemo_evaluator.sdk._executor.asyncio.to_thread", new=AsyncMock(), create=True) - executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) - - result = executor.run_local(spec=_EXACT_MATCH_EVALUATE_SPEC, workspace="ws") - - assert isinstance(result, EvaluatorLocalRunResult) - assert result.status == "completed" - assert result.artifact is not None - assert result.artifact.name == "evaluation-results" - assert result.artifact.artifact_url == "file:///results" - scheduler_cls.assert_called_once_with() - scheduler.run_local.assert_called_once_with( - EvaluateJob, - _EXACT_MATCH_EVALUATE_SPEC_JSON, - workspace="ws", - sdk=platform, - ) - to_thread.assert_not_called() - - class TestEvaluatorSubmit: """Tests for ``Evaluator.submit`` request construction.""" @@ -678,162 +617,6 @@ def test_requires_explicit_packager_for_custom_metric(self) -> None: ) -class TestEvaluatorRun: - """Tests for ``Evaluator.run`` executor delegation.""" - - def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> None: - """Run should forward the unpacked public kwargs to the executor.""" - platform = _SyncPlatform() - resource = Evaluator(cast(NeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - evaluate = mocker.patch.object(resource._executor, "evaluate", return_value=expected) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = [{"expected": "a", "output": "a"}] - - result = resource.run( - metric=metric, - dataset=dataset, - config=RunConfig(parallelism=2), - aggregate_fields=("mean", "max"), - ) - - assert result == expected - evaluate.assert_called_once_with( - metric=metric, - dataset=dataset, - params=RunConfig(parallelism=2), - target=None, - field_mapping=None, - prompt_template=None, - aggregate_fields=("mean", "max"), - ) - - def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: - """Run should forward FilesetRef datasets unchanged to the executor.""" - platform = _SyncPlatform() - resource = Evaluator(cast(NeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - evaluate = mocker.patch.object(resource._executor, "evaluate", return_value=expected) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = FilesetRef(root="default/helpsteer2") - - result = resource.run(metric=metric, dataset=dataset) - - assert result == expected - evaluate.assert_called_once_with( - metric=metric, - dataset=dataset, - params=None, - target=None, - field_mapping=None, - prompt_template=None, - aggregate_fields=None, - ) - - def test_run_uses_local_executor_execution(self, mocker: MockerFixture) -> None: - """Direct plugin SDK run should always use local executor execution.""" - platform = _SyncPlatform() - resource = Evaluator(cast(NeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - local_evaluate = mocker.patch.object(resource._executor, "evaluate", return_value=expected) - remote_evaluate = mocker.patch.object(resource._executor, "evaluate_remote") - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = [{"expected": "a", "output": "a"}] - - result = resource.run(metric=metric, dataset=dataset, aggregate_fields=("mean",)) - - assert result is expected - local_evaluate.assert_called_once_with( - metric=metric, - dataset=dataset, - params=None, - target=None, - field_mapping=None, - prompt_template=None, - aggregate_fields=("mean",), - ) - remote_evaluate.assert_not_called() - - -def test_sync_executor_evaluate_runs_local_job_with_packaged_input( - tmp_path: Path, - mocker: MockerFixture, -) -> None: - platform = _SyncPlatform() - executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - run_local = mocker.patch.object(executor, "run_local", return_value=_local_run_result(tmp_path, expected)) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = [{"expected": "a", "output": "a"}] - - result = executor.evaluate( - metric=metric, - dataset=dataset, - params=RunConfig(parallelism=2), - ) - - assert result == expected - run_local.assert_called_once() - assert run_local.call_args.kwargs["workspace"] == "platform-ws" - spec = run_local.call_args.kwargs["spec"] - assert isinstance(spec, EvaluateInputSpec) - assert _single_metric(spec).metric_type == "exact-match" - assert spec.dataset == dataset - assert spec.params == RunConfig(parallelism=2) - - -def test_sync_executor_evaluate_encodes_fileset_ref_before_local_job( - tmp_path: Path, - mocker: MockerFixture, -) -> None: - platform = _SyncPlatform() - executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - run_local = mocker.patch.object(executor, "run_local", return_value=_local_run_result(tmp_path, expected)) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = FilesetRef(root="default/helpsteer2#validation/*.jsonl") - - result = executor.evaluate( - metric=metric, - dataset=dataset, - ) - - assert result == expected - run_local.assert_called_once() - spec = run_local.call_args.kwargs["spec"] - assert isinstance(spec, EvaluateInputSpec) - assert spec.dataset == FilesetRef(root="default/helpsteer2#validation/*.jsonl") - - -def test_sync_executor_evaluate_remote_submits_waits_and_downloads(mocker: MockerFixture) -> None: - platform = _SyncPlatform() - executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - job_resource = mocker.Mock(spec=EvaluatorJobResource) - job_resource.get_result.return_value = expected - create = mocker.patch.object(executor, "create", return_value=job_resource) - result = executor.evaluate_remote( - metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - dataset=[{"expected": "a", "output": "a"}], - params=RunConfig(parallelism=2), - metric_bundle_packager=CloudpickleMetricBundlePackager(), - ) - - assert result == expected - create.assert_called_once() - assert create.call_args.kwargs["workspace"] == "platform-ws" - created_spec = create.call_args.kwargs["spec"] - assert _single_metric(created_spec).metric_type == "exact-match" - assert created_spec.dataset == [{"expected": "a", "output": "a"}] - assert created_spec.params == RunConfig(parallelism=2) - job_resource.wait_until_done.assert_called_once_with( - poll_interval_seconds=10.0, - job_timeout_seconds=3600.0, - pending_timeout_seconds=600.0, - ) - job_resource.get_result.assert_called_once_with(aggregate_fields=None) - - def test_sync_executor_submit_resolves_model_ref_before_creating_job(mocker: MockerFixture) -> None: platform = _SyncPlatform() executor = _SyncEvaluatorPluginExecutor(platform=cast(NeMoPlatform, platform)) @@ -931,7 +714,6 @@ async def test_async_executor_creates_evaluator_job(mocker: MockerFixture) -> No request=httpx.Request("POST", "http://test:8000/apis/evaluator/v2/workspaces/ws/evaluate/jobs"), json={"name": "job-123", "status": "created", "spec": _EXACT_MATCH_SPEC}, ) - to_thread = mocker.patch("nemo_evaluator.sdk._executor.asyncio.to_thread", new=AsyncMock(), create=True) http_client_cls = mocker.patch("nemo_evaluator.sdk._executor.httpx.Client") executor = _AsyncEvaluatorPluginExecutor(platform=cast(AsyncNeMoPlatform, platform)) spec = _EXACT_MATCH_EVALUATE_INPUT_SPEC @@ -949,7 +731,7 @@ async def test_async_executor_creates_evaluator_job(mocker: MockerFixture) -> No headers={"Authorization": "Bearer platform-token"}, timeout=platform.timeout, ) - to_thread.assert_not_awaited() + assert not hasattr(executor_module, "asyncio") http_client_cls.assert_not_called() @@ -1022,36 +804,6 @@ async def test_async_resource_url_encodes_reserved_chars_in_job_name() -> None: ) -@pytest.mark.asyncio -async def test_async_executor_runs_evaluator_job_locally_in_worker_thread(mocker: MockerFixture) -> None: - platform = _AsyncPlatform() - scheduler = mocker.Mock() - expected = {"status": "completed", "artifact": {"name": "evaluation-results", "artifact_url": "file:///results"}} - scheduler_cls = mocker.patch("nemo_evaluator.sdk._executor.NemoJobScheduler", return_value=scheduler, create=True) - mock_to_thread = mocker.patch( - "nemo_evaluator.sdk._executor.asyncio.to_thread", - new=AsyncMock(return_value=expected), - create=True, - ) - executor = _AsyncEvaluatorPluginExecutor(platform=cast(AsyncNeMoPlatform, platform)) - - result = await executor.run_local(spec=_EXACT_MATCH_EVALUATE_SPEC, workspace="ws") - - assert isinstance(result, EvaluatorLocalRunResult) - assert result.status == "completed" - assert result.artifact is not None - assert result.artifact.name == "evaluation-results" - assert result.artifact.artifact_url == "file:///results" - scheduler_cls.assert_called_once_with() - mock_to_thread.assert_awaited_once_with( - scheduler.run_local, - EvaluateJob, - _EXACT_MATCH_EVALUATE_SPEC_JSON, - workspace="ws", - async_sdk=platform, - ) - - class TestAsyncEvaluatorSubmit: """Tests for ``AsyncEvaluator.submit`` request construction.""" @@ -1174,86 +926,6 @@ async def test_requires_explicit_packager_for_custom_metric(self) -> None: ) -class TestAsyncEvaluatorRun: - """Tests for ``AsyncEvaluator.run`` executor delegation.""" - - @pytest.mark.asyncio - async def test_builds_request_from_unpacked_fields(self, mocker: MockerFixture) -> None: - """Run should forward the unpacked public kwargs to the executor.""" - platform = _AsyncPlatform() - resource = AsyncEvaluator(cast(AsyncNeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - evaluate = mocker.patch.object(resource._executor, "evaluate", new=AsyncMock(return_value=expected)) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = [{"expected": "a", "output": "a"}] - - result = await resource.run( - metric=metric, - dataset=dataset, - config=RunConfig(parallelism=2), - aggregate_fields=("mean", "max"), - ) - - assert result == expected - evaluate.assert_awaited_once_with( - metric=metric, - dataset=dataset, - params=RunConfig(parallelism=2), - target=None, - field_mapping=None, - prompt_template=None, - aggregate_fields=("mean", "max"), - ) - - @pytest.mark.asyncio - async def test_accepts_fileset_ref_dataset(self, mocker: MockerFixture) -> None: - """Run should forward FilesetRef datasets unchanged to the executor.""" - platform = _AsyncPlatform() - resource = AsyncEvaluator(cast(AsyncNeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - evaluate = mocker.patch.object(resource._executor, "evaluate", new=AsyncMock(return_value=expected)) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = FilesetRef(root="default/helpsteer2") - - result = await resource.run(metric=metric, dataset=dataset) - - assert result == expected - evaluate.assert_awaited_once_with( - metric=metric, - dataset=dataset, - params=None, - target=None, - field_mapping=None, - prompt_template=None, - aggregate_fields=None, - ) - - @pytest.mark.asyncio - async def test_run_uses_local_executor_execution(self, mocker: MockerFixture) -> None: - """Direct async plugin SDK run should always use local executor execution.""" - platform = _AsyncPlatform() - resource = AsyncEvaluator(cast(AsyncNeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - local_evaluate = mocker.patch.object(resource._executor, "evaluate", new=AsyncMock(return_value=expected)) - remote_evaluate = mocker.patch.object(resource._executor, "evaluate_remote", new=AsyncMock()) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = [{"expected": "a", "output": "a"}] - - result = await resource.run(metric=metric, dataset=dataset, aggregate_fields=("mean",)) - - assert result is expected - local_evaluate.assert_awaited_once_with( - metric=metric, - dataset=dataset, - params=None, - target=None, - field_mapping=None, - prompt_template=None, - aggregate_fields=("mean",), - ) - remote_evaluate.assert_not_awaited() - - @pytest.mark.asyncio async def test_async_executor_remote_submit_uses_platform_async_client_headers_and_timeout( mocker: MockerFixture, @@ -1279,69 +951,6 @@ async def test_async_executor_remote_submit_uses_platform_async_client_headers_a http_client_cls.assert_not_called() -@pytest.mark.asyncio -async def test_async_executor_evaluate_runs_local_job_with_packaged_input( - tmp_path: Path, - mocker: MockerFixture, -) -> None: - platform = _AsyncPlatform() - executor = _AsyncEvaluatorPluginExecutor(platform=cast(AsyncNeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - run_local = mocker.patch.object( - executor, - "run_local", - new=AsyncMock(return_value=_local_run_result(tmp_path, expected)), - ) - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = [{"expected": "a", "output": "a"}] - - result = await executor.evaluate( - metric=metric, - dataset=dataset, - params=RunConfig(parallelism=2), - ) - - assert result == expected - run_local.assert_awaited_once() - assert run_local.call_args.kwargs["workspace"] == "platform-ws" - spec = run_local.call_args.kwargs["spec"] - assert isinstance(spec, EvaluateInputSpec) - assert _single_metric(spec).metric_type == "exact-match" - assert spec.dataset == dataset - assert spec.params == RunConfig(parallelism=2) - - -@pytest.mark.asyncio -async def test_async_executor_evaluate_remote_submits_waits_and_downloads(mocker: MockerFixture) -> None: - platform = _AsyncPlatform() - executor = _AsyncEvaluatorPluginExecutor(platform=cast(AsyncNeMoPlatform, platform)) - expected = EvaluationResult(row_scores=[], aggregate_scores=AggregatedMetricResult(scores=[])) - job_resource = mocker.Mock(spec=AsyncEvaluatorJobResource) - job_resource.wait_until_done = AsyncMock() - job_resource.get_result = AsyncMock(return_value=expected) - create = mocker.patch.object(executor, "create", new=AsyncMock(return_value=job_resource)) - result = await executor.evaluate_remote( - metric=ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}"), - dataset=[{"expected": "a", "output": "a"}], - params=RunConfig(parallelism=2), - metric_bundle_packager=CloudpickleMetricBundlePackager(), - ) - - assert result == expected - create.assert_awaited_once() - assert create.call_args.kwargs["workspace"] == "platform-ws" - created_spec = create.call_args.kwargs["spec"] - assert _single_metric(created_spec).metric_type == "exact-match" - assert created_spec.dataset == [{"expected": "a", "output": "a"}] - assert created_spec.params == RunConfig(parallelism=2) - job_resource.wait_until_done.assert_awaited_once_with( - poll_interval_seconds=10.0, - job_timeout_seconds=3600.0, - pending_timeout_seconds=600.0, - ) - job_resource.get_result.assert_awaited_once_with(aggregate_fields=None) - - @pytest.mark.asyncio async def test_async_executor_submit_resolves_model_ref_before_creating_job(mocker: MockerFixture) -> None: platform = _AsyncPlatform() @@ -1379,21 +988,3 @@ async def test_async_executor_submit_rejects_online_params_without_target() -> N params=RunConfigOnline(), metric_bundle_packager=CloudpickleMetricBundlePackager(), ) - - -def test_local_run_result_requires_completed_artifact() -> None: - with pytest.raises(ValidationError): - EvaluatorLocalRunResult.model_validate({"status": "completed"}) - - -def test_local_run_result_allows_error_without_artifact_and_preserves_details() -> None: - result = EvaluatorLocalRunResult.model_validate({"status": "error", "message": "task failed"}) - - assert result.status == "error" - assert result.artifact is None - assert result.model_extra == {"message": "task failed"} - - -def test_local_run_result_rejects_unknown_status() -> None: - with pytest.raises(ValidationError): - EvaluatorLocalRunResult.model_validate({"status": "cancelled"}) From 4f33aca804d7ac8a91746001af221594d455922f Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 13 Aug 2026 09:58:58 -0300 Subject: [PATCH 2/3] docs(evaluator): move the plugin docs off the removed local run path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing `client.evaluator.run()` left 11 doc pages teaching it across 37 runnable snippets. Following any of them constructed the plugin resource and then raised `AttributeError`. The repo's own snippet linter proves it — `error[unresolved-attribute] Object of type Evaluator has no attribute run` — but it is not wired into CI, so nothing failed; the docs were simply wrong. Runnable snippets now submit, wait, and fetch: `job = evaluator.submit(...)`, `job.wait_until_done()`, `result = job.get_result()`. Two conversions needed more than a rename, because `run` and `submit` do not take the same arguments: `aggregate_fields` belongs on `get_result()` rather than `submit()`, and a `Model` target requires an online run config to match a `submit` overload. Prose that described local execution as a plugin mode now points at `nemo_evaluator_sdk.Evaluator`, which is where in-process evaluation lives. Two "Local execution" table rows are gone rather than reworded, since the mode they describe no longer exists. Left alone: the standalone SDK's own `Evaluator.run` / `run_sync`, which are a different class and still supported. Measured with `docs/_scripts/lint_python_snippets.py` against the branch point: diagnostics across the changed docs fall from 60 to 21, and no page regresses. Co-Authored-By: Claude Opus 5 Signed-off-by: Sandy Chapman --- docs/evaluator/agent-eval/index.mdx | 4 +- docs/evaluator/index.mdx | 18 ++- .../evaluator/metrics/agent-configuration.mdx | 6 +- docs/evaluator/metrics/agentic.mdx | 28 +++-- docs/evaluator/metrics/index.mdx | 6 +- docs/evaluator/metrics/llm-as-a-judge.mdx | 15 ++- docs/evaluator/metrics/manage-metrics.mdx | 6 +- .../evaluator/metrics/model-configuration.mdx | 29 +++-- docs/evaluator/metrics/rag.mdx | 46 ++++++-- docs/evaluator/metrics/remote.mdx | 13 ++- docs/evaluator/metrics/results.mdx | 6 +- docs/evaluator/metrics/similarity.mdx | 26 +++-- docs/evaluator/sdk-resources.mdx | 37 +----- docs/evaluator/test_doc_examples.py | 37 ++---- .../tutorials/run-llm-judge-evaluation.mdx | 12 +- plugins/nemo-auditor/src/nemo_auditor/sdk.py | 3 +- .../examples/plugin_examples.py | 105 ++++++------------ .../src/nemo_evaluator/jobs/evaluate.py | 6 +- .../src/nemo_evaluator/sdk/_executor.py | 49 +------- 19 files changed, 198 insertions(+), 254 deletions(-) diff --git a/docs/evaluator/agent-eval/index.mdx b/docs/evaluator/agent-eval/index.mdx index b095bdcf01..3194400bb4 100644 --- a/docs/evaluator/agent-eval/index.mdx +++ b/docs/evaluator/agent-eval/index.mdx @@ -17,8 +17,8 @@ got there*. Each task carries its own metrics, so a single suite can grade heter - Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs that do not require running nemo-platform. - Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline -tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces -above remain dataset-driven only. +tasks or stored tasksets. The high-level `client.evaluator.submit` interface +above remains dataset-driven only. diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index c21494fb3f..df51f55951 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -85,14 +85,13 @@ config = RunConfig(limit_samples=100, parallelism=8) ### 2. Run it — three dataset-driven modes -The same `metric`, `dataset`, and `config` run in three places. What changes is the **caller** — a -bare SDK evaluator for local iteration, or the platform's `client.evaluator` resource for -plugin-backed and durable execution. +The same `metric`, `dataset`, and `config` run in two places. What changes is the **caller** — a +bare SDK evaluator for local iteration, or the platform's `client.evaluator` resource for durable +execution. | Mode | Caller | Call | Best for | |------|--------|------|----------| -| **Local SDK** | `Evaluator()` (`nemo_evaluator_sdk`) | `await evaluator.run(metrics=metric, dataset=dataset, config=config)` | Fast in-process iteration with no platform services. | -| **Local plugin** | `client.evaluator` | `evaluator.run(metric=metric, dataset=dataset, config=config)` | Local runs through the platform runtime and Inference Gateway. | +| **Local SDK** | `Evaluator()` (`nemo_evaluator_sdk`) | `await evaluator.run(metrics=[metric], dataset=dataset, config=config)` | Fast in-process iteration with no platform services. | | **Remote job** | `client.evaluator` | `evaluator.submit(metric=metric, dataset=dataset, config=config)` | Durable, monitored platform jobs for production and regressions. | The platform caller is mounted on a `NeMoPlatform` client: @@ -109,9 +108,6 @@ client = NeMoPlatform( ) evaluator: Evaluator = client.evaluator -# Fast local iteration through the plugin runtime. -local_result = evaluator.run(metric=metric, dataset=dataset, config=config) - # Production evaluation as a durable platform job. job = evaluator.submit(metric=metric, dataset=dataset, config=config) job.wait_until_done() @@ -124,8 +120,8 @@ result = job.get_result() - Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs that do not require running nemo-platform. - Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline -tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces -above remain dataset-driven only. +tasks or stored tasksets. The high-level `client.evaluator.submit` interface +above remains dataset-driven only. @@ -136,7 +132,7 @@ the same and execution gains platform capabilities: | Capability | Local SDK | Platform (`client.evaluator`) | |------------|-----------|-------------------------------| -| **Execution** | Local in-process run | Local plugin runs, plus durable platform jobs | +| **Execution** | Local in-process run | Durable platform jobs | | **Inference** | Direct model or agent endpoint calls | The same, and can route through the NeMo Platform [Inference Gateway](/documentation/models-and-inference) and platform-managed endpoints | | **Datasets** | Inline rows and local files | Inline rows, local paths resolved at submission time, and NeMo Platform [Filesets](/documentation/get-started/core-concepts/manage-files) | | **Results** | Returned in memory | Platform artifact storage with typed result download | diff --git a/docs/evaluator/metrics/agent-configuration.mdx b/docs/evaluator/metrics/agent-configuration.mdx index 2b2cb7006f..d0b3ebea68 100644 --- a/docs/evaluator/metrics/agent-configuration.mdx +++ b/docs/evaluator/metrics/agent-configuration.mdx @@ -37,7 +37,7 @@ evaluator: Evaluator = client.evaluator # this object is an Evaluator resource If your agent endpoint requires authentication, configure `api_key_secret` on the `Agent`. -For local `evaluator.run(...)` calls, `api_key_secret` must name an environment variable available to the local Python process. For remote `evaluator.submit(...)` jobs, it must name a NeMo platform secret in the target workspace. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) for the local-versus-remote behavior. +For `evaluator.submit(...)` jobs, `api_key_secret` must name a NeMo platform secret in the target workspace. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). For remote `evaluator.submit(...)` jobs, create the secret in the platform workspace before submitting the job: @@ -89,7 +89,7 @@ agent = Agent( trajectory_path="$.reasoning_steps", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"question": "What is the capital of France?", "expected_answer": "Paris"}, @@ -98,6 +98,8 @@ result = evaluator.run( target=agent, prompt_template="Question: {{item.question}}\nAnswer:", ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` diff --git a/docs/evaluator/metrics/agentic.mdx b/docs/evaluator/metrics/agentic.mdx index ca668c1fc7..76dbb4eeaa 100644 --- a/docs/evaluator/metrics/agentic.mdx +++ b/docs/evaluator/metrics/agentic.mdx @@ -73,7 +73,7 @@ Agentic metrics evaluate different aspects of agent behavior: -Use `evaluator.run(...)` for local in-process evaluation and `evaluator.submit(...)` for durable remote platform jobs. The examples below use inline dataset rows through `dataset=[...]`, but you can use a file Path or a FilesetRef instead. +Use `evaluator.submit(...)` for durable platform jobs. For local in-process evaluation without the platform, use `nemo_evaluator_sdk.Evaluator` directly. The examples below use inline dataset rows through `dataset=[...]`, but you can use a file Path or a FilesetRef instead. ## Prerequisites @@ -135,7 +135,7 @@ from nemo_evaluator_sdk import ( ) ``` -Use `dataset=[...]` for inline rows. For offline scoring options, use `config=RunConfig(parallelism=...)`. Whenever outputs must be generated before scoring, pass `target=Model(...)` or `target=Agent(...)` plus the corresponding online parameters. Use the same `dataset`, `config`, and `target` arguments for both `evaluator.run(...)` and `evaluator.submit(...)`; durable jobs follow the identical pattern as local runs and only differ in waiting for and fetching results. +Use `dataset=[...]` for inline rows. For offline scoring options, use `config=RunConfig(parallelism=...)`. Whenever outputs must be generated before scoring, pass `target=Model(...)` or `target=Agent(...)` plus the corresponding online parameters. Pass `dataset`, `config`, and `target` to `evaluator.submit(...)`, then wait for the job and fetch its result. --- @@ -199,7 +199,7 @@ Evaluates whether the agent invoked the correct tools with the correct arguments from nemo_evaluator_sdk.metrics.ragas import ToolCallAccuracyMetric metric = ToolCallAccuracyMetric() -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -219,6 +219,8 @@ result = evaluator.run( } ], ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores) ``` @@ -374,7 +376,7 @@ Data must use OpenAI-compliant tool calling format: from nemo_evaluator_sdk import ToolCallingMetric metric = ToolCallingMetric(reference="{{item.tool_calls}}") -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -420,6 +422,8 @@ result = evaluator.run( } ], ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores) ``` @@ -557,7 +561,7 @@ metric = TopicAdherenceMetric( inference=InferenceParams.model_validate({"temperature": 0, "response_format": {"type": "json_object"}}), ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -572,6 +576,8 @@ result = evaluator.run( } ], ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores) ``` @@ -714,7 +720,7 @@ judge_model = Model( ) metric = AgentGoalAccuracyMetric(use_reference=True, judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -735,6 +741,8 @@ result = evaluator.run( } ], ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores) ``` @@ -861,7 +869,7 @@ judge_model = Model( ) metric = AgentGoalAccuracyMetric(use_reference=False, judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -890,6 +898,8 @@ result = evaluator.run( } ], ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores) ``` @@ -976,7 +986,7 @@ judge_model = Model( ) metric = AnswerAccuracyMetric(judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -986,6 +996,8 @@ result = evaluator.run( } ], ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores) ``` diff --git a/docs/evaluator/metrics/index.mdx b/docs/evaluator/metrics/index.mdx index f2110f754e..6395a36e8c 100644 --- a/docs/evaluator/metrics/index.mdx +++ b/docs/evaluator/metrics/index.mdx @@ -8,7 +8,7 @@ Metrics define how to score the outputs of your models, agents, or pipelines. ## What is a metric? -A metric is a scoring definition that evaluates model or agent outputs. In the Evaluator plugin SDK, metrics are inline Python objects passed directly to `evaluator.run(...)` or `evaluator.submit(...)`. +A metric is a scoring definition that evaluates model or agent outputs. In the Evaluator plugin SDK, metrics are inline Python objects passed directly to `evaluator.submit(...)`. - **Inputs**: For custom metrics, inputs define scoring logic composed of dataset fields and model outputs; for judge-based custom metrics, this also includes judge-model inputs (for example, judge prompts/rubrics and configuration). - **Outputs**: Row-level scores and aggregate statistics. @@ -57,13 +57,15 @@ evaluator: Evaluator = client.evaluator metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"expected": "Paris", "output": "Paris"}, {"expected": "Berlin", "output": "Munich"}, ], ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores) ``` diff --git a/docs/evaluator/metrics/llm-as-a-judge.mdx b/docs/evaluator/metrics/llm-as-a-judge.mdx index ef1fa666d5..ab7993655a 100644 --- a/docs/evaluator/metrics/llm-as-a-judge.mdx +++ b/docs/evaluator/metrics/llm-as-a-judge.mdx @@ -18,7 +18,6 @@ NeMo Evaluator supports two execution modes through the Evaluator plugin SDK: | Mode | Use Case | SDK Call | |------|----------|----------| -| **Local execution** | Rapid prototyping, metric development, and synchronous workflows | `evaluator.run(metric=metric, dataset=dataset)` | | **Durable remote job** | Production workloads that should run as platform jobs | `evaluator.submit(metric=metric, dataset=dataset)` | ## Prerequisites @@ -114,7 +113,7 @@ metric = LLMJudgeMetric( ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -127,6 +126,8 @@ result = evaluator.run( }, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean:.2f}, count={score.count}") @@ -222,7 +223,7 @@ metric = LLMJudgeMetric( }, ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -231,8 +232,9 @@ result = evaluator.run( }, {"input": "Explain quantum physics", "output": "I don't know."}, ], - aggregate_fields=("rubric_distribution", "mode_category"), ) +job.wait_until_done() +result = job.get_result(aggregate_fields=("rubric_distribution", "mode_category")) print(result.aggregate_scores.model_dump(exclude_none=True)) ``` @@ -244,14 +246,15 @@ By default, aggregate scores include `count`, `mean`, `min`, and `max`. Request ```python -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"input": "What is the capital of France?", "output": "Paris."}, {"input": "What is 2 + 2?", "output": "4."}, ], - aggregate_fields=("std_dev", "variance"), ) +job.wait_until_done() +result = job.get_result(aggregate_fields=("std_dev", "variance")) for score in result.aggregate_scores.scores: print(f"{score.name}:") diff --git a/docs/evaluator/metrics/manage-metrics.mdx b/docs/evaluator/metrics/manage-metrics.mdx index 0784c869c5..338cb48982 100644 --- a/docs/evaluator/metrics/manage-metrics.mdx +++ b/docs/evaluator/metrics/manage-metrics.mdx @@ -4,7 +4,7 @@ description: "" --- -Instantiate the metric class you want to run and pass it with `dataset` and optional configuration to `evaluator.run(...)` or `evaluator.submit(...)`. +Instantiate the metric class you want to run and pass it with `dataset` and optional configuration to `evaluator.submit(...)`. ## Initialize the SDK @@ -35,13 +35,15 @@ metric = ExactMatchMetric( candidate="{{item.output}}", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"expected": "Paris", "output": "Paris"}, {"expected": "Berlin", "output": "Munich"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") diff --git a/docs/evaluator/metrics/model-configuration.mdx b/docs/evaluator/metrics/model-configuration.mdx index 537a41e5f0..e826e69728 100644 --- a/docs/evaluator/metrics/model-configuration.mdx +++ b/docs/evaluator/metrics/model-configuration.mdx @@ -53,7 +53,7 @@ model = Model( `api_key_secret` is an optional property on the `Model` object. Omit it when the endpoint does not require API-key authentication. -For local `evaluator.run(...)` calls, `api_key_secret` must name an environment variable available to the local Python process. For example, `api_key_secret="NVIDIA_API_KEY"` reads `os.environ["NVIDIA_API_KEY"]`. +For `nemo_evaluator_sdk.Evaluator` runs, `api_key_secret` names an environment variable available to the local Python process. For example, `api_key_secret="NVIDIA_API_KEY"` reads `os.environ["NVIDIA_API_KEY"]`. For remote `evaluator.submit(...)` jobs, `api_key_secret` must name a NeMo platform secret in the target workspace. Create the secret before submitting the job: @@ -88,7 +88,7 @@ model = Model( metric = ExactMatchMetric(reference="{{item.expected_answer}}") -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"question": "What is the capital of France?", "expected_answer": "Paris"}, @@ -100,6 +100,8 @@ result = evaluator.run( target=model, prompt_template="Answer this question concisely: {{item.question}}", ) +job.wait_until_done() +result = job.get_result() ``` @@ -143,7 +145,7 @@ metric = LLMJudgeMetric( }, ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -153,6 +155,8 @@ result = evaluator.run( }, ], ) +job.wait_until_done() +result = job.get_result() ``` @@ -183,14 +187,14 @@ Use plain `RunConfig` for offline evaluations where the dataset already contains ## Model References -You can supply the evaluation target two ways. Which one is valid depends on whether you run the evaluation locally or submit it as a durable platform job. +You can supply the evaluation target two ways. Which one is valid depends on whether you submit the evaluation as a durable platform job or run it locally with `nemo_evaluator_sdk.Evaluator`. -### Inline `Model` (required for `evaluator.run(...)`) +### Inline `Model` -`evaluator.run(...)` executes in your local Python process, so it needs the resolved endpoint details inline. Always pass an inline `Model` as the `target` (or as a judge/embeddings field on the metric). If your deployment stores platform model entities, resolve the entity into endpoint details before constructing the `Model`: +An inline `Model` carries the resolved endpoint details. Always pass an inline `Model` as the `target` (or as a judge/embeddings field on the metric). If your deployment stores platform model entities, resolve the entity into endpoint details before constructing the `Model`: ```python -from nemo_evaluator_sdk import Model +from nemo_evaluator_sdk import Model, RunConfigOnlineModel model_entity = client.models.retrieve("my-model", workspace="default") model = Model( @@ -199,7 +203,14 @@ model = Model( api_key_secret="NVIDIA_API_KEY", ) -result = evaluator.run(metric=metric, dataset=dataset, target=model) +job = evaluator.submit( + metric=metric, + dataset=dataset, + config=RunConfigOnlineModel(parallelism=4), + target=model, +) +job.wait_until_done() +result = job.get_result() ``` ### `ModelRef` (supported by `evaluator.submit(...)`) @@ -217,7 +228,7 @@ job = evaluator.submit( ) ``` -`ModelRef` is **not** valid for `evaluator.run(...)`; the local runtime cannot resolve a platform entity. Pass an inline `Model` for local runs and either a `Model` or a `ModelRef` for remote submits. See the [Define and Run Custom Python Metrics](/documentation/evaluate-models/tutorials/define-and-run-custom-python-metrics) tutorial for an end-to-end `ModelRef` + `FilesetRef` submit example. +`evaluator.submit(...)` accepts either a `Model` or a `ModelRef`. `nemo_evaluator_sdk.Evaluator` resolves no platform entities, so pass it an inline `Model`. See the [Define and Run Custom Python Metrics](/documentation/evaluate-models/tutorials/define-and-run-custom-python-metrics) tutorial for an end-to-end `ModelRef` + `FilesetRef` submit example. diff --git a/docs/evaluator/metrics/rag.mdx b/docs/evaluator/metrics/rag.mdx index 4a9cb78e69..9060016fb7 100644 --- a/docs/evaluator/metrics/rag.mdx +++ b/docs/evaluator/metrics/rag.mdx @@ -58,7 +58,7 @@ client = NeMoPlatform( evaluator: Evaluator = client.evaluator # this object is an Evaluator resource ``` -Use `evaluator.run(metric=metric, dataset=dataset)` for a local synchronous evaluation. Use `evaluator.submit(metric=metric, dataset=dataset)` when you need a durable remote job: +Use `evaluator.submit(metric=metric, dataset=dataset)` for a durable platform job. For local in-process iteration without the platform, use `nemo_evaluator_sdk.Evaluator` directly: ```python def submit_evaluation(evaluator, metric, dataset): @@ -211,7 +211,9 @@ Measures the fraction of relevant content retrieved compared to the total releva ```python metric = ContextRecallMetric(judge_model=judge_model) -result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -287,7 +289,9 @@ Measures the proportion of relevant chunks in the retrieved contexts (precision@ ```python metric = ContextPrecisionMetric(judge_model=judge_model) -result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -357,7 +361,7 @@ Measures how relevant the retrieved contexts are to the user input. ```python metric = ContextRelevanceMetric(judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -367,6 +371,8 @@ result = evaluator.run( ], config=RunConfig(parallelism=8), ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -422,7 +428,7 @@ Measures how many important entities from the reference are present in the retri ```python metric = ContextEntityRecallMetric(judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -432,6 +438,8 @@ result = evaluator.run( ], config=RunConfig(parallelism=8), ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -488,7 +496,9 @@ Measures factual consistency of the response with the retrieved context. ```python metric = FaithfulnessMetric(judge_model=judge_model) -result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -500,13 +510,15 @@ for score in result.aggregate_scores.scores: ```python metric = FaithfulnessMetric(judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=online_dataset, config=online_config, target=generation_model, prompt_template=online_prompt_template, ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -559,7 +571,9 @@ Evaluates whether the response is grounded in the retrieved context without hall ```python metric = ResponseGroundednessMetric(judge_model=judge_model) -result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -571,13 +585,15 @@ for score in result.aggregate_scores.scores: ```python metric = ResponseGroundednessMetric(judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=online_dataset, config=online_config, target=generation_model, prompt_template=online_prompt_template, ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -639,7 +655,7 @@ Noise Sensitivity uses RAGAS mode-qualified outputs internally. The SDK normaliz ```python metric = NoiseSensitivityMetric(judge_model=judge_model) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -654,6 +670,8 @@ result = evaluator.run( ], config=RunConfig(parallelism=8), ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -725,7 +743,9 @@ metric = ResponseRelevancyMetric( strictness=1, ) -result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -741,13 +761,15 @@ metric = ResponseRelevancyMetric( strictness=1, ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=online_dataset, config=online_config, target=generation_model, prompt_template=online_prompt_template, ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") diff --git a/docs/evaluator/metrics/remote.mdx b/docs/evaluator/metrics/remote.mdx index 8886217f4d..f185665e1b 100644 --- a/docs/evaluator/metrics/remote.mdx +++ b/docs/evaluator/metrics/remote.mdx @@ -21,7 +21,6 @@ NeMo Evaluator supports two execution modes through the Evaluator plugin SDK: | Mode | Use Case | SDK Call | |------|----------|----------| -| **Local execution** | Rapid prototyping and synchronous workflows | `evaluator.run(metric=metric, dataset=dataset)` | | **Durable remote job** | Production workloads that should run as platform jobs | `evaluator.submit(metric=metric, dataset=dataset)` | ## Prerequisites @@ -85,13 +84,15 @@ metric = RemoteMetric( ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"reference": "The capital is Paris", "output": "Paris is the capital"}, {"reference": "2", "output": "2"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}, count={score.count}") @@ -117,7 +118,7 @@ metric = NemoAgentToolkitRemoteMetric( ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -131,6 +132,8 @@ result = evaluator.run( } ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -247,7 +250,9 @@ metric = RemoteMetric( api_key_secret=SecretRef(root="my-remote-api-key"), ) -result = evaluator.run(metric=metric, dataset=[{"input": "test"}]) +job = evaluator.submit(metric=metric, dataset=[{"input": "test"}]) +job.wait_until_done() +result = job.get_result() ``` diff --git a/docs/evaluator/metrics/results.mdx b/docs/evaluator/metrics/results.mdx index 07a1b333ea..a477a7d193 100644 --- a/docs/evaluator/metrics/results.mdx +++ b/docs/evaluator/metrics/results.mdx @@ -6,7 +6,7 @@ description: "" When a metric job completes, the platform automatically creates a **metric job result** — a persistent, queryable entity that captures the outcome of that run. Each result is created exactly once per completed job and shares the job's name. -Evaluator plugin SDK executions return an `EvaluationResult`. Synchronous local runs return it directly from `evaluator.run(...)`; durable remote jobs return it from `job.get_result()` after the job completes. +Evaluator plugin SDK executions return an `EvaluationResult` from `job.get_result()` once the submitted job completes. An `EvaluationResult` contains: @@ -31,13 +31,15 @@ evaluator: Evaluator = sdk.evaluator # this object is an Evaluator resource metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"expected": "Paris", "output": "Paris"}, {"expected": "Berlin", "output": "Munich"}, ], ) +job.wait_until_done() +result = job.get_result() ``` diff --git a/docs/evaluator/metrics/similarity.mdx b/docs/evaluator/metrics/similarity.mdx index 75138076e1..f50cf46e16 100644 --- a/docs/evaluator/metrics/similarity.mdx +++ b/docs/evaluator/metrics/similarity.mdx @@ -28,7 +28,7 @@ sdk = NeMoPlatform( evaluator: Evaluator = sdk.evaluator # this object is an Evaluator resource ``` -Use `evaluator.run(metric=metric, dataset=dataset)` for a local synchronous evaluation. Use `evaluator.submit(metric=metric, dataset=dataset)` when you need a durable remote job: +Use `evaluator.submit(metric=metric, dataset=dataset)` for a durable platform job. For local in-process iteration without the platform, use `nemo_evaluator_sdk.Evaluator` directly: ```python job = evaluator.submit(metric=metric, dataset=dataset) @@ -80,7 +80,7 @@ metric = BLEUMetric( candidate="{{item.model_output}}", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -95,6 +95,8 @@ result = evaluator.run( }, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -180,7 +182,7 @@ metric = ExactMatchMetric( description="Exact match for question answering", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"correct_answer": "Paris", "model_answer": "Paris"}, @@ -188,6 +190,8 @@ result = evaluator.run( {"correct_answer": "Berlin", "model_answer": "Munich"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -259,7 +263,7 @@ metric = F1Metric( candidate="{{item.answer}}", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -269,6 +273,8 @@ result = evaluator.run( {"reference": "a red apple", "answer": "red apple"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -352,7 +358,7 @@ metric = NumberCheckMetric( description="Check if values match within tolerance", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"expected": "100", "predicted": "100"}, @@ -360,6 +366,8 @@ result = evaluator.run( {"expected": "99", "predicted": "101"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -434,7 +442,7 @@ metric = ROUGEMetric( candidate="{{item.model_summary}}", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ { @@ -447,6 +455,8 @@ result = evaluator.run( }, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -545,7 +555,7 @@ metric = StringCheckMetric( right_template="{{item.must_contain}}", ) -result = evaluator.run( +job = evaluator.submit( metric=metric, dataset=[ {"output": "The answer is: 42", "must_contain": "answer"}, @@ -553,6 +563,8 @@ result = evaluator.run( {"output": "Error occurred", "must_contain": "Success"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") diff --git a/docs/evaluator/sdk-resources.mdx b/docs/evaluator/sdk-resources.mdx index ca0ee6d708..8a7dd57753 100644 --- a/docs/evaluator/sdk-resources.mdx +++ b/docs/evaluator/sdk-resources.mdx @@ -34,23 +34,11 @@ Use `submit` when you want to create a durable remote platform job and manage th | Method | Description | Returns | |--------|-------------|---------| -| `run()` | Runs one metric locally through the Evaluator plugin job runtime. | `EvaluationResult` | | `submit()` | Submits one metric evaluation as a durable platform job. | `EvaluatorJobResource` | | `plugin_status()` | Returns Evaluator plugin health information from the service. | `dict[str, object]` | | `get_job_resource(job_name: str, workspace: str \\| None = None)` | Returns a resource for an existing Evaluator plugin job. | `EvaluatorJobResource` | -The `dataset` argument accepts inline rows, local dataset paths, local glob paths, and fileset references with optional fragment selectors. Use `config` for evaluator runtime settings, `aggregate_fields` on result-returning calls to shape aggregate scores, and `target` plus `prompt_template` when the evaluator should generate model or agent responses before scoring. - -### `run()` arguments - -| Argument | Type | Required | Description | -|----------|------|----------|-------------| -| `metric` | `Metric` | Yes | Metric configuration used to score each row. | -| `dataset` | `PluginDatasetInput` | Yes | Inline rows, local dataset paths, local glob paths, or fileset references with optional fragment selectors. | -| `config` | `RunConfig \\| RunConfigOnline \\| RunConfigOnlineModel \\| None` | No | Runtime settings such as sample limits, parallelism, timeouts, and retry behavior. | -| `aggregate_fields` | `tuple[AggregateFieldName, ...] \\| None` | No | Aggregate score fields to include in the returned result. | -| `target` | `Model \\| Agent \\| None` | No | Model or agent target used when the evaluator should generate outputs before scoring. | -| `prompt_template` | `str \\| dict[str, Any] \\| None` | No | Prompt template used with `target` for online model or agent evaluation. | +The `dataset` argument accepts inline rows, local dataset paths, local glob paths, and fileset references with optional fragment selectors. Use `config` for evaluator runtime settings, and `target` plus `prompt_template` when the evaluator should generate model or agent responses before scoring. ### `submit()` arguments @@ -62,26 +50,6 @@ The `dataset` argument accepts inline rows, local dataset paths, local glob path | `target` | `Model \\| ModelRef \\| Agent \\| None` | No | Model, model reference, or agent target used when the submitted job should generate outputs before scoring. | | `prompt_template` | `str \\| dict[str, Any] \\| None` | No | Prompt template used with `target` for online model or agent evaluation. | -### Run locally - - - -```python -from nemo_evaluator_sdk import ExactMatchMetric - - -metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") -dataset = [ - {"expected": "Paris", "output": "Paris"}, - {"expected": "Berlin", "output": "Munich"}, -] - -result = evaluator.run(metric=metric, dataset=dataset) -print(result.aggregate_scores) -``` - - - ### Submit a platform job @@ -125,12 +93,11 @@ evaluator: AsyncEvaluator = client.evaluator | Method | Description | Returns | |--------|-------------|---------| -| `run()` | Runs one metric locally through the Evaluator plugin job runtime. | `EvaluationResult` | | `submit()` | Submits one metric evaluation as a durable platform job. | `AsyncEvaluatorJobResource` | | `plugin_status()` | Returns Evaluator plugin health information from the service. | `dict[str, object]` | | `get_job_resource(job_name: str, workspace: str \\| None = None)` | Returns a resource for an existing Evaluator plugin job. | `AsyncEvaluatorJobResource` | -`AsyncEvaluator.run()` and `AsyncEvaluator.submit()` accept the same arguments as the sync methods [above](#run-arguments). +`AsyncEvaluator.submit()` accepts the same arguments as the sync method [above](#submit-arguments). diff --git a/docs/evaluator/test_doc_examples.py b/docs/evaluator/test_doc_examples.py index 6126ee3601..b6a6a9239e 100644 --- a/docs/evaluator/test_doc_examples.py +++ b/docs/evaluator/test_doc_examples.py @@ -5,7 +5,7 @@ """Contract checks for the Evaluator SDK patterns used in these docs. The Evaluator docs are written against the ``nemo_evaluator`` plugin SDK -(``evaluator.run(...)`` / ``evaluator.submit(...)``), not the old +(``evaluator.submit(...)``), not the old ``/v2/.../evaluation/metrics/jobs`` REST endpoints. This module validates the import paths and call contract that every runnable doc snippet relies on, so the docs cannot silently drift from the SDK again. @@ -92,14 +92,18 @@ def _evaluator() -> Evaluator: return client.evaluator -def test_packager_param_is_submit_only() -> None: - """``submit`` takes ``metric_bundle_packager``; ``run`` (local, in-process) does not.""" +def test_packager_param_is_on_submit() -> None: + """``submit`` takes ``metric_bundle_packager``. + + This used to contrast ``submit`` against the local, in-process ``run``, which took no + packager because it never crossed a process boundary. That path is gone, so the surviving + half of the contract is that the packager rides on ``submit`` — and that ``run`` stays gone. + """ from nemo_evaluator.sdk import Evaluator submit_params = inspect.signature(Evaluator.submit).parameters - run_params = inspect.signature(Evaluator.run).parameters assert "metric_bundle_packager" in submit_params - assert "metric_bundle_packager" not in run_params + assert not hasattr(Evaluator, "run") def test_builtin_submit_does_not_require_a_packager() -> None: @@ -133,29 +137,6 @@ def test_custom_submit_requires_an_explicit_packager() -> None: evaluator.submit(metric=_CustomMetric(), dataset=dataset) -def test_run_does_not_require_metric_bundle_packager() -> None: - """``run()`` must not impose the submit-only packager requirement. - - ``run`` executes in-process; reaching the executor (which then needs a live - service) proves the packager guard did not fire. We only assert the failure - is NOT the packager ValueError. - """ - from nemo_evaluator_sdk import ExactMatchMetric - - evaluator = _evaluator() - metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.output}}") - dataset = [{"expected": "Paris", "output": "Paris"}] - - try: - evaluator.run(metric=metric, dataset=dataset) - except ValueError as error: # pragma: no cover - defensive - assert "metric_bundle_packager is required" not in str(error) - except Exception: - # Any non-ValueError (e.g. connection error to the local runtime) is fine; - # it means we got past argument validation. - pass - - def main() -> None: raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx index 3441481280..e84f67b6c4 100644 --- a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx +++ b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx @@ -60,7 +60,7 @@ Before you begin, here is a quick overview of the resources you will use: - **Fileset**: A dataset registered with NeMo Platform. The evaluator plugin SDK accepts fileset references directly, so this tutorial passes the registered HelpSteer2 split to evaluations as a `FilesetRef`. - **Workspace**: A workspace that isolates your resources. Secrets, filesets, and jobs belong to a workspace. - **Job**: A durable remote platform task created with `evaluator.submit(...)`. -- **Evaluation**: The process of scoring model outputs using one or more metrics. Use `evaluator.run(...)` for local in-process execution, `evaluator.submit(...)` for durable jobs +- **Evaluation**: The process of scoring model outputs using one or more metrics. Use `evaluator.submit(...)` for durable jobs, or `nemo_evaluator_sdk.Evaluator` for local in-process execution --- @@ -163,7 +163,7 @@ from nemo_evaluator.sdk import FilesetRef JUDGE_MODEL_URL = "https://integrate.api.nvidia.com/v1/chat/completions" JUDGE_MODEL_NAME = "nvidia/nemotron-3-nano-30b-a3b" -# Local evaluator.run() resolves this value from os.environ["NVIDIA_API_KEY"]. +# A submitted job resolves this name against platform secrets in the workspace. LOCAL_JUDGE_MODEL = Model( url=JUDGE_MODEL_URL, name=JUDGE_MODEL_NAME, @@ -307,11 +307,11 @@ Use low temperature for evaluation tasks. Low or zero temperature produces outpu ## 6. Test with Local Evaluation -Before running a durable job, test your metric with a few examples using `evaluator.run(...)`. This runs locally in-process and returns results immediately, which is useful for prompt iteration. +Before running a durable job, test your metric with a few examples using `nemo_evaluator_sdk.Evaluator`. It runs locally in-process and returns results immediately, which is useful for prompt iteration. ```python -quick_test_result = evaluator.run( +job = evaluator.submit( metric=metric_v1_local, dataset=[ { @@ -325,6 +325,8 @@ quick_test_result = evaluator.run( ], config=RunConfig(parallelism=1), ) +job.wait_until_done() +quick_test_result = job.get_result() def score_value(row_score, score_name: str) -> float | None: @@ -656,7 +658,7 @@ To delete the workspace, you must first delete all resources within it. Delete j ```python from nemo_platform import NotFoundError -# Delete remote evaluation jobs. Local evaluator.run() results are in-memory +# Delete remote evaluation jobs. Local SDK results are in-memory # objects and do not create platform jobs. for job_name in [job_v1.name, job_v2.name]: try: diff --git a/plugins/nemo-auditor/src/nemo_auditor/sdk.py b/plugins/nemo-auditor/src/nemo_auditor/sdk.py index 0fa75c72cd..58a0406339 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/sdk.py +++ b/plugins/nemo-auditor/src/nemo_auditor/sdk.py @@ -16,8 +16,7 @@ - ``client.auditor.list_jobs(workspace=...)`` — list submitted audit jobs. - ``client.auditor.get_job(job_name, workspace=...)`` — fetch a single audit job. - ``client.auditor.run(config=..., target=..., workspace=...)`` — in-process - audit using :class:`~nemo_auditor.jobs.audit.AuditJob`. Mirrors the evaluator - plugin's ``client.evaluator.run`` pattern: delegates to + audit using :class:`~nemo_auditor.jobs.audit.AuditJob`. Delegates to :meth:`~nemo_platform_plugin.scheduler.NemoJobScheduler.run_local`, which constructs a tempdir-backed :class:`~nemo_platform_plugin.job_context.JobContext` and writes report artifacts via diff --git a/plugins/nemo-evaluator/examples/plugin_examples.py b/plugins/nemo-evaluator/examples/plugin_examples.py index 982440d764..f7f4b5ef1a 100644 --- a/plugins/nemo-evaluator/examples/plugin_examples.py +++ b/plugins/nemo-evaluator/examples/plugin_examples.py @@ -14,7 +14,7 @@ from collections.abc import Sequence from pathlib import Path from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING, Any, Literal, cast +from typing import TYPE_CHECKING, Any, cast from nemo_evaluator.jobs.evaluate import EvaluateSpec from nemo_evaluator.sdk import FilesetRef @@ -65,7 +65,6 @@ 'Return only a JSON object with this shape: {"helpfulness": }.' ) ONLINE_CHAT_PROMPT_TEMPLATE = {"messages": [{"role": "user", "content": "{{item.prompt}}"}]} -ExampleExecutionMode = Literal["run", "submit"] LOCAL_HELPSTEER2_ROWS = ( { "prompt": "What is the capital of France?", @@ -280,15 +279,12 @@ async def ensure_submit_evaluator_api_key_secret(workspace: str, client: AsyncNe async def model_with_valid_secret( *, - execution_mode: ExampleExecutionMode, workspace: str, client: AsyncNeMoPlatform, ) -> Model: - """Return a model configured for run or submit NeMo Platform example execution.""" - if execution_mode == "submit": - secret_name = await ensure_submit_evaluator_api_key_secret(workspace, client) - return model.model_copy(update={"api_key_secret": SecretRef(root=secret_name)}) - return model + """Return a model carrying the API-key secret that a submitted platform job needs.""" + secret_name = await ensure_submit_evaluator_api_key_secret(workspace, client) + return model.model_copy(update={"api_key_secret": SecretRef(root=secret_name)}) def create_helpfulness_metric(judge_model: Model) -> LLMJudgeMetric: @@ -389,21 +385,12 @@ def _assert_exact_match_result(result: EvaluationResult, *, workflow: str, expec async def _evaluate_metric( evaluator_plugin_client: AsyncEvaluator, *, - execution_mode: ExampleExecutionMode, metric: Metric, dataset: PluginDatasetInput, config: RunConfig | RunConfigOnlineModel, **run_kwargs: Any, ) -> EvaluationResult: - """Run or submit based on the requested plugin SDK execution mode.""" - if execution_mode == "run": - return await evaluator_plugin_client.run( - metric=metric, - dataset=dataset, - config=config, - **run_kwargs, - ) - + """Submit the metric as a platform job and wait for its result.""" job = await evaluator_plugin_client.submit( metric=metric, dataset=dataset, @@ -463,7 +450,6 @@ async def _run_online_metric_example_body( dataset: PluginDatasetInput, workflow_label: str, is_online: bool, - execution_mode: ExampleExecutionMode, limit_samples: int, ) -> None: """Evaluate one exact-match metric against an already-built dataset. @@ -480,7 +466,6 @@ async def _run_online_metric_example_body( metric = _online_exact_match_metric() config = RunConfigOnlineModel(parallelism=4, limit_samples=limit_samples) run_kwargs["target"] = await model_with_valid_secret( - execution_mode=execution_mode, workspace=DEFAULT_WORKSPACE, client=client, ) @@ -488,7 +473,6 @@ async def _run_online_metric_example_body( result = await _evaluate_metric( evaluator_plugin_client, - execution_mode=execution_mode, metric=metric, dataset=dataset, config=config, @@ -498,7 +482,7 @@ async def _run_online_metric_example_body( if not is_online: _assert_exact_match_result( result, - workflow=f"{execution_mode} {workflow_label}", + workflow=workflow_label, expected_rows=limit_samples, ) else: @@ -507,14 +491,12 @@ async def _run_online_metric_example_body( async def run_nmp_online_metric_example( is_online: bool = False, - execution_mode: ExampleExecutionMode = "run", limit_samples: int = 2, ) -> None: """Evaluate one metric through the plugin SDK using run or submit.""" _print_example_separator( run_nmp_online_metric_example.__name__, is_online=is_online, - execution_mode=execution_mode, limit_samples=limit_samples, ) client = await _new_client() @@ -525,7 +507,6 @@ async def run_nmp_online_metric_example( dataset=dataset, workflow_label="exact-match", is_online=is_online, - execution_mode=execution_mode, limit_samples=limit_samples, ) finally: @@ -534,14 +515,12 @@ async def run_nmp_online_metric_example( def run_nmp_online_metric_example_sync_client( is_online: bool = False, - execution_mode: ExampleExecutionMode = "run", limit_samples: int = 2, ) -> None: """Evaluate one metric through the plugin SDK using a sync platform client.""" _print_example_separator( run_nmp_online_metric_example_sync_client.__name__, is_online=is_online, - execution_mode=execution_mode, limit_samples=limit_samples, ) client = _new_sync_client() @@ -558,35 +537,27 @@ def run_nmp_online_metric_example_sync_client( run_kwargs["target"] = model run_kwargs["prompt_template"] = ONLINE_CHAT_PROMPT_TEMPLATE - if execution_mode == "run": - result = evaluator_plugin_client.run( - metric=metric, - dataset=dataset, - config=config, - **run_kwargs, - ) - else: - job = evaluator_plugin_client.submit( - metric=metric, - dataset=dataset, - config=config, - metric_bundle_packager=CloudpickleMetricBundlePackager(), - **run_kwargs, - ) - print(f"Submitted evaluator plugin job: {job.name}") - job.wait_until_done( - poll_interval_seconds=1, - job_timeout_seconds=300, - pending_timeout_seconds=120, - ) - result = job.get_result() - artifacts_dir = job.download_artifacts(path="evaluation_artifacts") - print(f"Saved artifacts under {artifacts_dir}") + job = evaluator_plugin_client.submit( + metric=metric, + dataset=dataset, + config=config, + metric_bundle_packager=CloudpickleMetricBundlePackager(), + **run_kwargs, + ) + print(f"Submitted evaluator plugin job: {job.name}") + job.wait_until_done( + poll_interval_seconds=1, + job_timeout_seconds=300, + pending_timeout_seconds=120, + ) + result = job.get_result() + artifacts_dir = job.download_artifacts(path="evaluation_artifacts") + print(f"Saved artifacts under {artifacts_dir}") if not is_online: _assert_exact_match_result( result, - workflow=f"sync {execution_mode} exact-match", + workflow="sync submit exact-match", expected_rows=limit_samples, ) else: @@ -597,14 +568,12 @@ def run_nmp_online_metric_example_sync_client( async def run_nmp_online_metric_local_file_example( is_online: bool = False, - execution_mode: ExampleExecutionMode = "run", limit_samples: int = 2, ) -> None: """Evaluate one metric through the plugin SDK using a local JSONL Path dataset.""" _print_example_separator( run_nmp_online_metric_local_file_example.__name__, is_online=is_online, - execution_mode=execution_mode, limit_samples=limit_samples, ) client = await _new_client() @@ -617,7 +586,6 @@ async def run_nmp_online_metric_local_file_example( dataset=dataset_path, workflow_label="local-file exact-match", is_online=is_online, - execution_mode=execution_mode, limit_samples=limit_samples, ) finally: @@ -627,14 +595,12 @@ async def run_nmp_online_metric_local_file_example( async def run_nmp_llm_judge_example( is_online: bool = False, limit_samples: int = 2, - execution_mode: ExampleExecutionMode = "run", ) -> None: """Evaluate a helpfulness judge through the plugin SDK using run or submit.""" _print_example_separator( run_nmp_llm_judge_example.__name__, is_online=is_online, limit_samples=limit_samples, - execution_mode=execution_mode, ) client = await _new_client() @@ -642,7 +608,6 @@ async def run_nmp_llm_judge_example( dataset = await ensure_example_fileset(client) run_kwargs: dict[str, Any] = {} judge_model = await model_with_valid_secret( - execution_mode=execution_mode, workspace=DEFAULT_WORKSPACE, client=client, ) @@ -656,7 +621,6 @@ async def run_nmp_llm_judge_example( result = await _evaluate_metric( evaluator_plugin_client, - execution_mode=execution_mode, metric=create_helpfulness_metric(judge_model), dataset=dataset, config=config, @@ -677,23 +641,28 @@ async def run_nmp_llm_judge_example( async def run_examples(*, include_submit: bool = False, include_model_calls: bool = False) -> None: - """Execute the example workflows exposed by this module.""" - await run_nmp_online_metric_example(is_online=False, execution_mode="run") - await run_nmp_online_metric_local_file_example(is_online=False, execution_mode="run") + """Execute the example workflows exposed by this module. - if include_submit: - await run_nmp_online_metric_example(is_online=False, execution_mode="submit") + Every workflow here submits a platform job, so all of them are gated behind + ``include_submit``. The examples that used to run without one went through the plugin's + local execution path, which no longer exists — use ``nemo_evaluator_sdk.Evaluator`` for + evaluation that does not need a running platform. + """ + if not include_submit: + print("All plugin examples submit platform jobs; pass --include-submit to run them.") + return + + await run_nmp_online_metric_example(is_online=False) + await run_nmp_online_metric_local_file_example(is_online=False) if include_model_calls: - await run_nmp_llm_judge_example(is_online=False, execution_mode="run") - if include_submit: - await run_nmp_llm_judge_example(is_online=True, execution_mode="submit") + await run_nmp_llm_judge_example(is_online=True) def run_sync_examples(*, include_submit: bool = False) -> None: """Execute the synchronous example workflows exposed by this module.""" if include_submit: - run_nmp_online_metric_example_sync_client(is_online=False, execution_mode="submit") + run_nmp_online_metric_example_sync_client(is_online=False) def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index c62c9fab2b..df12d953c7 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -261,8 +261,10 @@ async def to_spec( workspace: str, entity_client: object, # Widened from the base signature: `resolve_metrics_to_inline` documents that it takes - # either client, and the local-run path (`_executor._resolve_sync_local_spec`) forwards the - # sync one. Contravariant, so overriding with a wider parameter stays substitutable. + # either client. Contravariant, so overriding with a wider parameter stays substitutable. + # The caller that actually forwarded a sync client was the plugin's local-run path, now + # removed, so this could likely narrow to the async client alone — left alone here to keep + # this change a pure deletion. async_sdk: AsyncNeMoPlatform | NeMoPlatform | None, is_local: bool, ) -> BaseModel: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py index 4b5062b28a..df51256708 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py @@ -6,12 +6,12 @@ from __future__ import annotations from collections.abc import Sequence -from typing import Any, TypeAlias, cast +from typing import Any import httpx from nemo_evaluator.api.schemas import MetricInline from nemo_evaluator.filesets import FilesetRef -from nemo_evaluator.jobs.evaluate import EvaluateInputSpec, EvaluateJob, EvaluateSpec, TargetSpec +from nemo_evaluator.jobs.evaluate import EvaluateInputSpec, TargetSpec from nemo_evaluator.resolvers import PlatformModelResolver from nemo_evaluator.sdk import http_utils from nemo_evaluator.sdk.job_resources import ( @@ -46,7 +46,6 @@ _DEFAULT_JOB_TIMEOUT_SECONDS = 3600.0 _DEFAULT_PENDING_TIMEOUT_SECONDS = 600.0 -EvaluateRequestSpec: TypeAlias = EvaluateInputSpec | EvaluateSpec SubmitTargetSpec = TargetSpec | ModelRef @@ -141,50 +140,6 @@ def _build_evaluate_spec( return EvaluateInputSpec.model_validate(spec) -def _resolve_sync_local_spec( - spec: EvaluateRequestSpec, - *, - platform: NeMoPlatform, - workspace: str, -) -> EvaluateSpec: - """Return a canonical local spec, resolving input-only model references with the sync SDK.""" - if isinstance(spec, EvaluateSpec): - return spec - return cast( - EvaluateSpec, - run_sync( - lambda: EvaluateJob.to_spec( - spec, - workspace=workspace, - entity_client=None, - async_sdk=platform, - is_local=True, - ) - ), - ) - - -async def _resolve_async_local_spec( - spec: EvaluateRequestSpec, - *, - platform: AsyncNeMoPlatform, - workspace: str, -) -> EvaluateSpec: - """Return a canonical local spec, resolving input-only model references with the async SDK.""" - if isinstance(spec, EvaluateSpec): - return spec - return cast( - EvaluateSpec, - await EvaluateJob.to_spec( - spec, - workspace=workspace, - entity_client=None, - async_sdk=platform, - is_local=True, - ), - ) - - class _SyncEvaluatorPluginExecutor: """Sync evaluator plugin executor used by the sync SDK resource.""" From 3e5ae432202e10af7fc1bcd10ff863a089723457 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Thu, 13 Aug 2026 10:08:46 -0300 Subject: [PATCH 3/3] docs(evaluator): correct what the run-to-submit conversion changed underneath An independent review caught that converting `run` to `submit` silently changed what `api_key_secret` means. Under local execution it named an environment variable; a submitted job resolves it as a secret in the job workspace. The converted snippets kept `NVIDIA_API_KEY` while the surrounding prose tells you to create a platform secret called `nvidia-api-key`, so following the page end-to-end would fail model authentication. Submit-bound models now name the platform secret. The judge tutorial had the sharper version of the same problem: section 6 is titled "Test with Local Evaluation" and keeps a deliberately local model, but the conversion had it submitting a platform job. It now uses `nemo_evaluator_sdk.Evaluator`, which is what the section is for and what its model's env-var secret expects. Also: six agentic tabs still said "Run Locally" over snippets that now submit and block; the SDK resources page still advertised `run` as a primary execution method in prose; and the example's `--include-submit` help still described running "in addition to run-mode examples" when nothing runs without it. Snippet diagnostics across the changed docs fall from 85 to 35, no page regresses. Co-Authored-By: Claude Opus 5 Signed-off-by: Sandy Chapman --- docs/evaluator/metrics/agentic.mdx | 14 +++++++------- docs/evaluator/metrics/model-configuration.mdx | 8 ++++---- docs/evaluator/sdk-resources.mdx | 7 +++---- .../tutorials/run-llm-judge-evaluation.mdx | 9 ++++----- plugins/nemo-evaluator/examples/plugin_examples.py | 2 +- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/docs/evaluator/metrics/agentic.mdx b/docs/evaluator/metrics/agentic.mdx index 76dbb4eeaa..c34be6daad 100644 --- a/docs/evaluator/metrics/agentic.mdx +++ b/docs/evaluator/metrics/agentic.mdx @@ -193,7 +193,7 @@ Evaluates whether the agent invoked the correct tools with the correct arguments - + ```python from nemo_evaluator_sdk.metrics.ragas import ToolCallAccuracyMetric @@ -368,7 +368,7 @@ Data must use OpenAI-compliant tool calling format: - + ```python @@ -535,13 +535,13 @@ Measures how well the agent maintained focus on assigned topics throughout a con -Topic Adherence is a multi-turn metric: it scores a complete conversation supplied as a `user_input` message list. Online target generation produces a single response from a single prompt and cannot construct a multi-turn conversation, so it is not supported for this metric. Use the offline **Run Locally** or **Submit Job** modes with a pre-built multi-turn conversation. +Topic Adherence is a multi-turn metric: it scores a complete conversation supplied as a `user_input` message list. Online target generation produces a single response from a single prompt and cannot construct a multi-turn conversation, so it is not supported for this metric. Use the offline **Offline Scoring** or **Submit Job** modes with a pre-built multi-turn conversation. - + ```python from nemo_evaluator_sdk import InferenceParams, Model, SecretRef @@ -707,7 +707,7 @@ Compare the agent's outcome against a known reference: - + ```python from nemo_evaluator_sdk import Model, SecretRef @@ -856,7 +856,7 @@ The judge LLM infers the goal from the conversation context: - + ```python from nemo_evaluator_sdk import Model, SecretRef @@ -973,7 +973,7 @@ Evaluates the factual correctness of an agent's answer by comparing it against a - + ```python from nemo_evaluator_sdk import Model, SecretRef diff --git a/docs/evaluator/metrics/model-configuration.mdx b/docs/evaluator/metrics/model-configuration.mdx index e826e69728..0a104760d4 100644 --- a/docs/evaluator/metrics/model-configuration.mdx +++ b/docs/evaluator/metrics/model-configuration.mdx @@ -36,7 +36,7 @@ model = Model( url="https://integrate.api.nvidia.com/v1", name="meta/llama-3.1-70b-instruct", format="nim", - api_key_secret="NVIDIA_API_KEY", + api_key_secret="nvidia-api-key", ) ``` @@ -83,7 +83,7 @@ model = Model( url="https://integrate.api.nvidia.com/v1", name="meta/llama-3.1-70b-instruct", format="nim", - api_key_secret="NVIDIA_API_KEY", + api_key_secret="nvidia-api-key", ) metric = ExactMatchMetric(reference="{{item.expected_answer}}") @@ -119,7 +119,7 @@ judge_model = Model( url="https://integrate.api.nvidia.com/v1", name="meta/llama-3.1-70b-instruct", format="nim", - api_key_secret="NVIDIA_API_KEY", + api_key_secret="nvidia-api-key", ) metric = LLMJudgeMetric( model=judge_model, @@ -200,7 +200,7 @@ model_entity = client.models.retrieve("my-model", workspace="default") model = Model( url=client.models.get_model_entity_route_openai_url(model_entity), name="my-model", - api_key_secret="NVIDIA_API_KEY", + api_key_secret="nvidia-api-key", ) job = evaluator.submit( diff --git a/docs/evaluator/sdk-resources.mdx b/docs/evaluator/sdk-resources.mdx index 8a7dd57753..cb118287f1 100644 --- a/docs/evaluator/sdk-resources.mdx +++ b/docs/evaluator/sdk-resources.mdx @@ -7,7 +7,7 @@ description: "" The `nemo_evaluator_sdk` package provides context-agnostic objects for defining metrics, datasets, evaluation configuration, and result handling. When you want to execute those evaluations through the NeMo Platform Evaluator plugin, use the Evaluator SDK resource mounted on the `nemo_platform` SDK. -This page explains the NeMo Platform-specific objects used to run local plugin jobs, submit durable platform jobs, and retrieve evaluator job results. +This page explains the NeMo Platform-specific objects used to submit durable platform jobs and retrieve evaluator job results. ## Evaluator @@ -28,9 +28,8 @@ client = NeMoPlatform( evaluator: Evaluator = client.evaluator # this object is an Evaluator resource ``` -The primary execution methods are `run` and `submit`. -Use `run` when you want a local in-process plugin execution that returns a completed `EvaluationResult`. -Use `submit` when you want to create a durable remote platform job and manage the job lifecycle separately. +The primary execution method is `submit`, which creates a durable remote platform job whose lifecycle you manage through the returned job resource. +For local in-process evaluation that returns a completed `EvaluationResult` without the platform, use `nemo_evaluator_sdk.Evaluator` directly. | Method | Description | Returns | |--------|-------------|---------| diff --git a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx index e84f67b6c4..7f9c8ec1d6 100644 --- a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx +++ b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx @@ -163,7 +163,7 @@ from nemo_evaluator.sdk import FilesetRef JUDGE_MODEL_URL = "https://integrate.api.nvidia.com/v1/chat/completions" JUDGE_MODEL_NAME = "nvidia/nemotron-3-nano-30b-a3b" -# A submitted job resolves this name against platform secrets in the workspace. +# `nemo_evaluator_sdk.Evaluator` resolves this name from your local environment. LOCAL_JUDGE_MODEL = Model( url=JUDGE_MODEL_URL, name=JUDGE_MODEL_NAME, @@ -310,9 +310,10 @@ Use low temperature for evaluation tasks. Low or zero temperature produces outpu Before running a durable job, test your metric with a few examples using `nemo_evaluator_sdk.Evaluator`. It runs locally in-process and returns results immediately, which is useful for prompt iteration. ```python +from nemo_evaluator_sdk import Evaluator as LocalEvaluator -job = evaluator.submit( - metric=metric_v1_local, +quick_test_result = LocalEvaluator().run_sync( + metrics=[metric_v1_local], dataset=[ { "prompt": "What is the capital of France?", @@ -325,8 +326,6 @@ job = evaluator.submit( ], config=RunConfig(parallelism=1), ) -job.wait_until_done() -quick_test_result = job.get_result() def score_value(row_score, score_name: str) -> float | None: diff --git a/plugins/nemo-evaluator/examples/plugin_examples.py b/plugins/nemo-evaluator/examples/plugin_examples.py index f7f4b5ef1a..48c5015720 100644 --- a/plugins/nemo-evaluator/examples/plugin_examples.py +++ b/plugins/nemo-evaluator/examples/plugin_examples.py @@ -671,7 +671,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument( "--include-submit", action="store_true", - help="Submit evaluator jobs in addition to run-mode examples.", + help="Run the example workflows. Every one submits an evaluator job, so without this flag none of them run.", ) parser.add_argument( "--include-model-calls",