feat(evaluator): add GymRunnerTarget plugin wiring and Studio rendering

plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py

@@ -33,6 +33,7 @@
     AgentTarget,
     CodexRunnerTarget,
     FabricRunnerTarget,
+    GymRunnerTarget,
     HarborRunnerTarget,
     ModelTarget,
     Target,
@@ -46,6 +47,7 @@
 from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult
 from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime
 from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
+from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner, GymRuntimeConfig
 from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner, HarborRuntimeConfig
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTarget
@@ -350,6 +352,24 @@ def _resolve_target(
                 work_root=ctx.storage.persistent / "fabric",
             )
             return fabric_runtime, None, None
+        if isinstance(target, GymRunnerTarget):
+            gym_runtime = GymAgentTaskRunner(
+                config=GymRuntimeConfig(
+                    agent=target.agent,
+                    agent_config=target.agent_config,
+                    resources_server=target.resources_server,
+                    model_type=target.model_type,
+                    bind_resources_server=target.bind_resources_server,
+                    env_overrides=target.env_overrides,
+                    num_repeats=target.num_repeats,
+                    concurrency=target.concurrency,
+                    startup_timeout_s=target.startup_timeout_s,
+                    collection_timeout_s=target.collection_timeout_s,
+                    shutdown_grace_s=target.shutdown_grace_s,
+                    reward_key=target.reward_key,
+                )
+            )
+            return gym_runtime, None, None
         if isinstance(target, HarborRunnerTarget):
             harbor_runtime = HarborAgentTaskRunner(
                 config=HarborRuntimeConfig(

plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py

@@ -144,9 +144,62 @@ class HarborRunnerTarget(BaseModel):
     )
 
 
+class GymRunnerTarget(BaseModel):
+    """Generate trials by driving a NeMo Gym environment through the SDK's :class:`GymAgentTaskRunner`.
+
+    Gym runs locally in the job container (the ``gym`` CLI must be installed in the same environment
+    as this SDK). The environment dataset is recovered from the tasks at run time — the runner stamps
+    ``gym_dataset_path`` onto each task via ``discover_gym_tasks``, mirroring the Harbor pattern.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    kind: Literal["gym"] = "gym"
+    agent: str = Field(description="Agent name to collect rollouts with, e.g. 'simple_agent'.")
+    agent_config: str = Field(
+        description="Repo-relative agent config passed to `gym env start` (--config).",
+    )
+    resources_server: str = Field(
+        description="Resources-server (environment) name, e.g. 'mcqa' (--resources-server).",
+    )
+    model_type: str = Field(
+        default="inference_provider",
+        description="Model-type config (--model-type). `inference_provider` speaks OpenAI-compatible chat; "
+        "`openai_model` uses the OpenAI Responses API.",
+    )
+    bind_resources_server: bool = Field(
+        default=True,
+        description="Auto-bind the agent's `resources_server.name` via a Hydra override. Set False for "
+        "self-contained agents that already bind their own resources-server.",
+    )
+    env_overrides: list[str] = Field(
+        default_factory=list,
+        description="Extra Hydra '+key=value' overrides for `gym env start` (applied after the auto-derived "
+        "resources-server binding).",
+    )
+    num_repeats: int = Field(default=1, ge=1, description="Attempts per row; each attempt becomes one trial.")
+    concurrency: int = Field(
+        default=4,
+        ge=1,
+        description="Concurrent rollouts for `gym eval run`.",
+    )
+    startup_timeout_s: float = Field(default=240.0, gt=0, description="Max wait for `gym env start` readiness.")
+    collection_timeout_s: float | None = Field(
+        default=None,
+        gt=0,
+        description="Max wait for `gym eval run` collection; None = unbounded.",
+    )
+    shutdown_grace_s: float = Field(
+        default=30.0,
+        gt=0,
+        description="Grace period for the Gym subprocess group to exit on SIGTERM before escalating to SIGKILL.",
+    )
+    reward_key: str = Field(default="reward", description="Key read from each rollout record.")
+
+
 #: The agent-runner slot of the target union — the spec-side mirror of ``AgentTaskRunner``, resolved
 #: to a runtime at run time. ``kind``-discriminated; widen with more members as runners land.
-AgentRunnerTarget: TypeAlias = CodexRunnerTarget | FabricRunnerTarget | HarborRunnerTarget
+AgentRunnerTarget: TypeAlias = CodexRunnerTarget | FabricRunnerTarget | GymRunnerTarget | HarborRunnerTarget
 
 #: What generates trials: a Model or Agent endpoint, or an agent runner. ``kind``-discriminated, and
 #: the spec-level analog of the SDK's runtime ``AgentEvalTarget`` (Model | Agent | AgentTaskRunner).
@@ -168,6 +221,8 @@ def target_agent_identity(target: Target | None) -> tuple[str | None, str | None
         return target.agent.name, None
     if isinstance(target, HarborRunnerTarget):
         return target.agent_import_path or target.agent_name, target.agent_model_name
+    if isinstance(target, GymRunnerTarget):
+        return target.agent, None
     if isinstance(target, ModelTarget):
         return None, target.model.name
     if isinstance(target, CodexRunnerTarget | FabricRunnerTarget):

plugins/nemo-evaluator/src/nemo_evaluator/jobs/result_persistence.py

@@ -24,6 +24,7 @@
     AgentTarget,
     CodexRunnerTarget,
     FabricRunnerTarget,
+    GymRunnerTarget,
     HarborRunnerTarget,
     ModelTarget,
     Target,
@@ -92,6 +93,8 @@ def _agent_target_fields(target: Target | None) -> tuple[str | None, str | None,
         return "codex", target.model, None
     if isinstance(target, FabricRunnerTarget):
         return "fabric", target.model, None
+    if isinstance(target, GymRunnerTarget):
+        return "gym", target.agent, None
     if isinstance(target, HarborRunnerTarget):
         return "harbor", target.agent_import_path or target.agent_name, None
     return None, None, None

plugins/nemo-evaluator/tests/jobs/test_publication.py

@@ -23,6 +23,7 @@
     AgentTarget,
     CodexRunnerTarget,
     FabricRunnerTarget,
+    GymRunnerTarget,
     HarborRunnerTarget,
     IntakePublicationSpec,
     ModelTarget,
@@ -177,6 +178,10 @@ def _publish(client: AsyncNeMoPlatform | None, *, required: bool = True, agent_n
         (ModelTarget(model=Model(name="gpt-4o", url="http://model")), (None, "gpt-4o")),
         (HarborRunnerTarget(agent_name="oracle", agent_model_name="m"), ("oracle", "m")),
         (HarborRunnerTarget(agent_name="oracle", agent_import_path="pkg:Agent"), ("pkg:Agent", None)),
+        (
+            GymRunnerTarget(agent="simple_agent", agent_config="conf/agent.yaml", resources_server="mcqa"),
+            ("simple_agent", None),
+        ),
         (CodexRunnerTarget(model="gpt-5.5"), (None, "gpt-5.5")),
         (FabricRunnerTarget(config={}, model="p/m"), (None, "p/m")),
         (None, (None, None)),
@@ -226,6 +231,17 @@ def test_agent_name_derived_from_agent_target_needs_no_override() -> None:
     assert target_agent_identity(spec.target)[0] == "derived"
 
 
+def test_agent_name_derived_from_gym_target_needs_no_override() -> None:
+    spec = _input_spec(
+        GymRunnerTarget(agent="simple_agent", agent_config="conf/agent.yaml", resources_server="mcqa"),
+        PublicationSpec(intake=IntakePublicationSpec(evaluation_id="eval-1")),
+    )
+    assert spec.publication is not None
+    assert spec.publication.intake is not None
+    assert spec.publication.intake.agent_name is None
+    assert target_agent_identity(spec.target)[0] == "simple_agent"
+
+
 @pytest.mark.parametrize(
     "target",
     [

plugins/nemo-evaluator/tests/test_agent_evaluate.py

@@ -28,6 +28,7 @@
     AgentTarget,
     CodexRunnerTarget,
     FabricRunnerTarget,
+    GymRunnerTarget,
     HarborRunnerTarget,
     ModelTarget,
     Target,
@@ -40,6 +41,7 @@
 from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary
 from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime
 from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
+from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner
 from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import (
@@ -303,6 +305,27 @@ def test_resolve_target_resolves_none_to_no_target(tmp_path: Path) -> None:
     assert AgentEvalJob._resolve_target(None, ctx) == (None, None, None)
 
 
+def test_resolve_target_builds_gym_runtime_from_runner_target(tmp_path: Path) -> None:
+    ctx = _job_context(tmp_path)
+    gym_target = GymRunnerTarget(
+        agent="simple_agent",
+        agent_config="responses_api_agents/simple_agent/configs/simple_agent.yaml",
+        resources_server="mcqa",
+        num_repeats=2,
+        concurrency=4,
+        reward_key="score",
+    )
+    target, prompt_template, params = AgentEvalJob._resolve_target(gym_target, ctx)
+    assert isinstance(target, GymAgentTaskRunner)
+    assert target._config.agent == "simple_agent"
+    assert target._config.resources_server == "mcqa"
+    assert target._config.num_repeats == 2
+    assert target._config.reward_key == "score"
+    # A runner shapes its own request, so it contributes no prompt template or inference params.
+    assert prompt_template is None
+    assert params is None
+
+
 def test_runner_target_is_accepted(tmp_path: Path) -> None:
     spec = AgentEvalSpec(tasks=[_task_spec()], target=CodexRunnerTarget(model="gpt-5.5"))
     assert isinstance(spec.target, CodexRunnerTarget)
@@ -732,6 +755,11 @@ async def test_compile_rejects_reserved_secret_env_name() -> None:
         AgentTarget(agent=_agent(), params=RunConfigOnline()),
         CodexRunnerTarget(model="gpt-5.5"),
         HarborRunnerTarget(agent_name="oracle"),
+        GymRunnerTarget(
+            agent="simple_agent",
+            agent_config="responses_api_agents/simple_agent/configs/simple_agent.yaml",
+            resources_server="mcqa",
+        ),
     ],
 )
 def test_run_local_executes_each_target_type(target: Target, mocker: MockerFixture) -> None:
@@ -758,6 +786,8 @@ def test_run_local_executes_each_target_type(target: Target, mocker: MockerFixtu
         assert isinstance(fake.received_target, CodexCliAgentRuntime)
     elif isinstance(target, HarborRunnerTarget):
         assert isinstance(fake.received_target, HarborAgentTaskRunner)
+    elif isinstance(target, GymRunnerTarget):
+        assert isinstance(fake.received_target, GymAgentTaskRunner)
     elif isinstance(target, ModelTarget):
         assert getattr(fake.received_target, "name", None) == target.model.name
     else:

plugins/nemo-evaluator/tests/test_result_persistence.py

@@ -19,6 +19,7 @@
     AgentTarget,
     CodexRunnerTarget,
     FabricRunnerTarget,
+    GymRunnerTarget,
     HarborRunnerTarget,
     ModelTarget,
 )
@@ -83,6 +84,10 @@ def _agent() -> Agent:
             FabricRunnerTarget(config={"metadata": {"name": "a"}}, model="openai/gpt-5.4"),
             ("fabric", "openai/gpt-5.4", None),
         ),
+        (
+            GymRunnerTarget(agent="simple_agent", agent_config="conf/agent.yaml", resources_server="mcqa"),
+            ("gym", "simple_agent", None),
+        ),
         (HarborRunnerTarget(agent_name="oracle"), ("harbor", "oracle", None)),
         (HarborRunnerTarget(agent_import_path="wrapper:Agent"), ("harbor", "wrapper:Agent", None)),
         (None, (None, None, None)),

web/packages/studio/src/components/evaluation/EvalAggregateScoresTable.tsx

@@ -23,6 +23,9 @@ export interface EvalAggregateScoreRow {
 interface EvalAggregateScoresTableProps {
   scores: EvalAggregateScoreRow[];
   emptyMessage?: string;
+  /** When true, score badges are rendered gray instead of the traffic-light color scale.
+   *  Use for runner-contributed metrics whose scale is runner-defined and not comparable to [0, 1]. */
+  disableScoreColoring?: boolean;
 }
 
 const VIEW_PREFIX = 'view.';
@@ -47,6 +50,7 @@ const displayScoreValue = (score: EvalAggregateScoreRow): number | null =>
 export const EvalAggregateScoresTable: FC<EvalAggregateScoresTableProps> = ({
   scores,
   emptyMessage = 'No scores recorded for this evaluation.',
+  disableScoreColoring = false,
 }) => {
   const dataViewState = useStudioDataViewState();
   const hasRubric = scores.some((score) => !!score.rubric_distribution?.length);
@@ -78,7 +82,10 @@ export const EvalAggregateScoresTable: FC<EvalAggregateScoresTableProps> = ({
         enableSorting: false,
         size: 100,
         cell: ({ row }) => (
-          <Badge kind="solid" color={scoreColor(displayScoreValue(row.original))}>
+          <Badge
+            kind="solid"
+            color={disableScoreColoring ? 'gray' : scoreColor(displayScoreValue(row.original))}
+          >
             {formatScore(displayScoreValue(row.original))}
           </Badge>
         ),
@@ -130,7 +137,7 @@ export const EvalAggregateScoresTable: FC<EvalAggregateScoresTableProps> = ({
           ]
         : []),
     ],
-    [hasRubric]
+    [hasRubric, disableScoreColoring]
   );
 
   if (scores.length === 0) {