diff --git a/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx b/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx index 5d2c9fbc74..ab3ff85589 100644 --- a/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx +++ b/web/packages/studio/src/components/evaluation/SubmitEvaluationModal.tsx @@ -2,8 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { zodResolver } from '@hookform/resolvers/zod'; -import { ControlledDatasetFileSelect } from '@nemo/common/src/components/DatasetFileSelect/ControlledDatasetFileSelect'; -import { parseFilesetLocation } from '@nemo/common/src/components/DatasetFileSelect/parseFilesetLocation'; import { ControlledSelect } from '@nemo/common/src/components/form/ControlledSelect'; import { ControlledTextInput } from '@nemo/common/src/components/form/ControlledTextInput'; import { FormModal, type FormModalProps } from '@nemo/common/src/components/FormModal'; @@ -19,10 +17,13 @@ import type { EvaluateJobRequest, } from '@nemo/sdk/generated/evaluator/schema'; import { + createExperiment, + deleteExperiment, filesCreateFileset, filesDeleteFileset, filesDownloadFile, filesUploadFile, + useListExperiments, } from '@nemo/sdk/generated/platform/api'; import { Anchor, @@ -35,6 +36,13 @@ import { import { fetchSampleText } from '@studio/api/agents/fetchSampleText'; import { submitAgentEvalJob } from '@studio/api/evaluation/agent-evaluations'; import { isConflictError, type EvalSeedFile } from '@studio/api/evaluation/eval-config-fileset'; +import { + createRunEvaluation, + EVAL_CONFIG_FILENAME, + EVAL_CONFIG_FILESET_KEY, + experimentConfigError, + experimentFilesetName, +} from '@studio/components/evaluation/experimentEvalConfig'; import { JudgeModelSelect } from '@studio/components/evaluation/JudgeModelSelect'; import { bareName, @@ -42,12 +50,13 @@ import { buildDatasetEvalRequestBody, buildPersistedSpec, type EvalSpec, + filesetNameForExperiment, injectJudgeModel, type InlineMetricBundle, isDatasetEvalSpec, generateEvalConfigName, MODE_DEFAULT, - MODE_FILESET, + MODE_EXPERIMENT, parseEvalConfig, } from '@studio/components/evaluation/submitEvaluationJob'; import { LINK_EVAL_DOCS_APPROACHES } from '@studio/constants/links'; @@ -69,12 +78,13 @@ import { z } from 'zod'; const EVAL_CONFIG_MODE_ITEMS = [ { value: MODE_DEFAULT, children: 'Use Example' }, - { value: MODE_FILESET, children: 'Choose Fileset' }, + { value: MODE_EXPERIMENT, children: 'Choose Experiment' }, ]; -/** Flat filename the reusable config is stored as inside its fileset. */ -const EVAL_CONFIG_FILENAME = 'eval-config.json'; const DATASET_FILENAME = 'dataset.jsonl'; + +/** Backend caps page_size at 100; the picker shows the most recent page. */ +const EXPERIMENT_PAGE_SIZE = 100; const README_FILENAME = 'README.md'; const NO_DEPLOYMENT_MESSAGE = 'This agent has no active deployment.'; @@ -84,10 +94,14 @@ const DEPLOYMENT_CHECK_FAILED_MESSAGE = const submitEvaluationBaseSchema = z.object({ agent: z.string().min(1, 'Agent is required'), judgeModel: z.string(), - mode: z.enum([MODE_DEFAULT, MODE_FILESET]), + mode: z.enum([MODE_DEFAULT, MODE_EXPERIMENT]), exampleKey: z.string(), + /** Name of the experiment to create in "Use Example" mode. */ newName: z.string(), - configFile: z.string().nullable(), + /** Fileset created alongside it, holding eval-config.json and any data artifacts. */ + filesetName: z.string(), + /** Name of the experiment to re-run in "Choose Experiment" mode. */ + experimentName: z.string(), }); type SubmitEvaluationFormData = z.infer; @@ -110,12 +124,20 @@ const makeSubmitEvaluationSchema = (requiresJudgeModel: () => boolean) => path: ['newName'], }); } + const filesetError = getEntityNameError(data.filesetName.trim()); + if (filesetError) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: filesetError, + path: ['filesetName'], + }); + } } - if (data.mode === MODE_FILESET && !parseFilesetLocation(data.configFile ?? '')?.objectPath) { + if (data.mode === MODE_EXPERIMENT && !data.experimentName) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: 'Pick an eval-config.json inside an existing fileset', - path: ['configFile'], + message: 'Pick an experiment to run', + path: ['experimentName'], }); } }); @@ -128,14 +150,18 @@ const makeSubmitEvaluationSchema = (requiresJudgeModel: () => boolean) => const runningDeploymentsQuery = (agent: string): AgentsListDeploymentsParams => ({ filter: { agent: bareName(agent), status: 'running' } }) as AgentsListDeploymentsParams; -const makeDefaultValues = (agent?: string): SubmitEvaluationFormData => ({ - agent: agent ?? '', - judgeModel: '', - mode: MODE_DEFAULT, - exampleKey: DEFAULT_EVAL_CONFIG_KEY, - newName: generateEvalConfigName(), - configFile: null, -}); +const makeDefaultValues = (agent?: string): SubmitEvaluationFormData => { + const newName = generateEvalConfigName(); + return { + agent: agent ?? '', + judgeModel: '', + mode: MODE_DEFAULT, + exampleKey: DEFAULT_EVAL_CONFIG_KEY, + newName, + filesetName: filesetNameForExperiment(newName), + experimentName: '', + }; +}; interface SubmitEvaluationModalProps extends Pick { workspace: string; @@ -145,17 +171,51 @@ interface SubmitEvaluationModalProps extends Pick void; } +/** Undo what a failed submit created, and return the error to raise. + * + * Everything here is name-unique per workspace, so a leftover holds the name the retry wants: + * without this, the second attempt fails on a conflict instead of the original problem. + * Deleting the Experiment also soft-deletes the Evaluation created under it (the API cascades + * to members whose only membership was that group) and frees both names, so the two deletes + * below unwind the whole chain. When a delete itself fails the returned error names what to + * remove by hand. */ +const discardSeeded = async ( + workspace: string, + seeded: { filesetName?: string; experimentName?: string }, + cause: unknown +): Promise => { + const leftovers: string[] = []; + const signal = new AbortController().signal; + if (seeded.experimentName) { + await deleteExperiment(workspace, seeded.experimentName, signal).catch(() => + leftovers.push(`experiment "${seeded.experimentName}"`) + ); + } + if (seeded.filesetName) { + await filesDeleteFileset(workspace, seeded.filesetName, signal).catch(() => + leftovers.push(`fileset "${seeded.filesetName}"`) + ); + } + if (leftovers.length === 0) return cause; + const causeDetail = cause instanceof Error ? cause.message : String(cause); + return new Error( + `${causeDetail} — ${leftovers.join(' and ')} could not be removed; delete manually before retrying under the same name.`, + { cause } + ); +}; + /** Resolves the persisted yardstick spec for this submission. In "Use Example" mode * it builds the spec from the sample template (fanning the metric onto every task with * the picked judge baked in) and seeds it into a new fileset; in "Choose Fileset" mode * it reads the saved spec back verbatim (no re-fan, no judge re-pick). */ const loadPersistedSpec = async ( workspace: string, - formData: SubmitEvaluationFormData + formData: SubmitEvaluationFormData, + experimentFileset: string | null ): Promise => { if (formData.mode === MODE_DEFAULT) { const signal = new AbortController().signal; - const name = formData.newName.trim(); + const name = formData.filesetName.trim(); const example = getEvalConfigSample(formData.exampleKey); const template = parseEvalConfig(await fetchSampleText(example.configPath)); const files: EvalSeedFile[] = []; @@ -217,30 +277,21 @@ const loadPersistedSpec = async ( ); } } catch (uploadErr) { - try { - await filesDeleteFileset(workspace, name, signal); - } catch (cleanupErr) { - const uploadDetail = uploadErr instanceof Error ? uploadErr.message : String(uploadErr); - const cleanupDetail = cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr); - throw new Error( - `${uploadDetail} — the partially created fileset "${name}" could not be removed (${cleanupDetail}); delete it before retrying.`, - { cause: uploadErr } - ); - } - throw uploadErr; + throw await discardSeeded(workspace, { filesetName: name }, uploadErr); } return spec; } - // Choose-fileset mode: read the saved yardstick spec out of its fileset, as-is. - const parsed = parseFilesetLocation(formData.configFile ?? ''); - if (!parsed?.objectPath) throw new Error('No eval-config.json selected'); + // Choose-experiment mode: read the saved spec out of the experiment's own fileset, as-is. + // The fileset is reached through the experiment, never picked directly, so there is no + // per-run file choice to make — the config lives at a known path by convention. + if (!experimentFileset) throw new Error('The selected experiment has no eval config fileset'); const blob = await filesDownloadFile( workspace, - parsed.name, - parsed.objectPath, + experimentFileset, + EVAL_CONFIG_FILENAME, new AbortController().signal ); - if (!blob) throw new Error('Failed to read the selected eval config'); + if (!blob) throw new Error("Failed to read the selected experiment's eval config"); return parseEvalConfig(await blob.text()); }; @@ -278,7 +329,6 @@ export const SubmitEvaluationModal: FC = ({ setValue, getValues, handleSubmit, - setError, clearErrors, formState, } = methods; @@ -309,6 +359,35 @@ export const SubmitEvaluationModal: FC = ({ const agentFieldError = errors.agent?.message ?? deploymentError; const exampleKey = useWatch({ control, name: 'exampleKey' }); + const experimentName = useWatch({ control, name: 'experimentName' }); + + // Experiments to re-run. There is no "metadata key exists" filter, so the ones without a + // config fileset can only be excluded after the fact — see experimentConfigError below. + const { data: experimentsResponse, isLoading: isExperimentsLoading } = useListExperiments( + workspace, + { page_size: EXPERIMENT_PAGE_SIZE, sort: '-created_at' }, + { query: { enabled: open && mode === MODE_EXPERIMENT } } + ); + const experiments = experimentsResponse?.data ?? []; + const selectedExperiment = experiments.find((item) => item.name === experimentName); + + // Validate on selection rather than at submit: a bad pick should be obvious before the user + // commits, and the check is two cheap reads. + const { data: experimentConfigIssue, isFetching: isValidatingExperiment } = useQuery({ + queryKey: ['experiment-eval-config', workspace, experimentName], + queryFn: ({ signal }) => + selectedExperiment ? experimentConfigError(workspace, selectedExperiment, signal) : null, + enabled: open && mode === MODE_EXPERIMENT && !!selectedExperiment, + }); + + const experimentFileset = selectedExperiment ? experimentFilesetName(selectedExperiment) : null; + const experimentFieldError = errors.experimentName?.message ?? experimentConfigIssue ?? undefined; + + // Hold submit while the pick is still being checked, so a bad experiment cannot slip through + // the gap between selecting it and the validation landing. + const canRunSelectedExperiment = + mode !== MODE_EXPERIMENT || + (!isValidatingExperiment && !!selectedExperiment && !experimentConfigIssue); // Fetch and parse the selected example config early to detect metric type and default model. const { data: exampleConfig } = useQuery({ @@ -365,23 +444,58 @@ export const SubmitEvaluationModal: FC = ({ reset: resetMutation, } = useMutation({ mutationFn: async (formData: SubmitEvaluationFormData) => { - const spec = await loadPersistedSpec(workspace, formData); - const filesetName = - formData.mode === MODE_DEFAULT - ? formData.newName.trim() - : (parseFilesetLocation(formData.configFile ?? '')?.name ?? undefined); - const selections = { workspace, agent: formData.agent, filesetName }; - const created = isDatasetEvalSpec(spec) - ? await evaluatorCreateEvaluateJob( - workspace, - buildDatasetEvalRequestBody(spec, selections, null) as EvaluateJobRequest - ) - : await submitAgentEvalJob( - workspace, - buildAgentEvalRequestBody(spec, selections) as AgentEvaluateJobRequest - ); - if (!created?.name) throw new Error('Submission did not return a job name'); - return { name: created.name, isDataset: isDatasetEvalSpec(spec) }; + const spec = await loadPersistedSpec(workspace, formData, experimentFileset); + + // Order is forced by the backend: the fileset is seeded above, then the Experiment must + // exist before an Evaluation can reference it, and the Evaluation before the job can + // publish to it — the worker creates neither. + const isNew = formData.mode === MODE_DEFAULT; + const filesetName = isNew ? formData.filesetName.trim() : (experimentFileset ?? ''); + + // What this submit created, so a failure rolls back exactly that and nothing pre-existing. + const seeded: { filesetName?: string; experimentName?: string } = isNew + ? { filesetName } + : {}; + + try { + const experiment = isNew + ? await createExperiment(workspace, { + name: formData.newName.trim(), + metadata: { [EVAL_CONFIG_FILESET_KEY]: filesetName }, + }) + : selectedExperiment; + if (!experiment) throw new Error('No experiment to run this evaluation under'); + if (isNew) seeded.experimentName = experiment.name; + + const evaluationId = await createRunEvaluation(workspace, { + experimentId: experiment.id, + experimentName: experiment.name, + filesetName, + }); + + const selections = { + workspace, + agent: formData.agent, + filesetName, + experimentName: experiment.name, + evaluationId, + }; + const created = isDatasetEvalSpec(spec) + ? await evaluatorCreateEvaluateJob( + workspace, + buildDatasetEvalRequestBody(spec, selections, null) as EvaluateJobRequest + ) + : await submitAgentEvalJob( + workspace, + buildAgentEvalRequestBody(spec, selections) as AgentEvaluateJobRequest + ); + if (!created?.name) throw new Error('Submission did not return a job name'); + return { name: created.name, isDataset: isDatasetEvalSpec(spec) }; + } catch (err) { + // Re-running an existing experiment seeds nothing, so there is nothing to unwind: its + // fileset predates this submit and its Evaluation is reused by the retry. + throw await discardSeeded(workspace, seeded, err); + } }, onSuccess: ({ name, isDataset }) => { toast.success(`Evaluation "${name}" submitted`); @@ -435,7 +549,7 @@ export const SubmitEvaluationModal: FC = ({ submitButtonText="Submit" onSubmit={handleSubmit(onSubmit)} disabled={isPending} - submitDisabled={!deploymentVerified} + submitDisabled={!deploymentVerified || !canRunSelectedExperiment} loading={isPending} errorText={errorMessage} className="w-[690px]! max-w-[95vw]!" @@ -475,10 +589,10 @@ export const SubmitEvaluationModal: FC = ({ className="w-full [&_button]:flex-1" value={mode} onValueChange={(v) => { - setValue('mode', v as typeof MODE_DEFAULT | typeof MODE_FILESET, { + setValue('mode', v as typeof MODE_DEFAULT | typeof MODE_EXPERIMENT, { shouldValidate: false, }); - clearErrors('configFile'); + clearErrors('experimentName'); }} items={EVAL_CONFIG_MODE_ITEMS} /> @@ -530,31 +644,35 @@ export const SubmitEvaluationModal: FC = ({ useControllerProps={{ control, name: 'newName' }} selectOnFocus formFieldProps={{ - slotLabel: 'New Fileset Name', - slotHelp: 'Saves a reusable eval-config.json you can select for future runs.', + slotLabel: 'New Experiment Name', + slotHelp: + 'Groups this run and future ones against the same config. Select it later to re-run.', slotError: errors.newName?.message, }} /> + ) : ( - + item.name ? [{ value: item.name, children: item.name }] : [] + )} + formFieldProps={{ + slotLabel: 'Experiment', + slotHelp: `Runs the ${EVAL_CONFIG_FILENAME} in the experiment's fileset.`, + slotError: experimentFieldError, + status: experimentFieldError ? 'error' : undefined, }} - acceptedFileTypes={['.json']} - invalidFileMode="disable" - setError={(error) => setError('configFile', error)} - clearError={() => clearErrors('configFile')} - workspace={workspace} - inline - autoCommit - autoSelectFirstAcceptable - showUpdatedAt - filesetPurpose="generic" - datasetLabel="Fileset" - formFieldProps={{ slotError: errors.configFile?.message }} /> )} diff --git a/web/packages/studio/src/components/evaluation/experimentEvalConfig.ts b/web/packages/studio/src/components/evaluation/experimentEvalConfig.ts new file mode 100644 index 0000000000..d9c13a8ea5 --- /dev/null +++ b/web/packages/studio/src/components/evaluation/experimentEvalConfig.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createEvaluation, filesListFilesetFiles } from '@nemo/sdk/generated/platform/api'; +import type { ExperimentResponse } from '@nemo/sdk/generated/platform/schema'; +import { buildEvalJobName } from '@studio/components/evaluation/submitEvaluationJob'; + +/** Experiment metadata key holding the name of the fileset that stores its eval config. + * Metadata values are plain strings, which is all a fileset name needs to be. */ +export const EVAL_CONFIG_FILESET_KEY = 'eval_config_fileset'; + +/** Flat filename the reusable config is stored as inside its fileset. Every Experiment's + * fileset must carry one at the root — there is no per-run file picker. */ +export const EVAL_CONFIG_FILENAME = 'eval-config.json'; + +/** The fileset an Experiment stores its eval config in, or null when it names none. */ +export const experimentFilesetName = (experiment: ExperimentResponse): string | null => + experiment.metadata?.[EVAL_CONFIG_FILESET_KEY] ?? null; + +/** Why an Experiment cannot be run against, or null when it can. Two ways to be invalid: + * it names no fileset, or the fileset it names has no eval-config.json at the root. */ +export const experimentConfigError = async ( + workspace: string, + experiment: ExperimentResponse, + signal?: AbortSignal +): Promise => { + const filesetName = experimentFilesetName(experiment); + if (!filesetName) { + return `Experiment "${experiment.name}" has no eval config fileset. Pick another experiment, or create one from a template.`; + } + const files = await filesListFilesetFiles(workspace, filesetName, undefined, signal).catch( + () => null + ); + if (!files) { + return `Could not read fileset "${filesetName}" for experiment "${experiment.name}". Pick another experiment.`; + } + const hasConfig = (files.data ?? []).some((file) => file.path === EVAL_CONFIG_FILENAME); + return hasConfig + ? null + : `Fileset "${filesetName}" has no ${EVAL_CONFIG_FILENAME} at its root, so experiment "${experiment.name}" cannot be run. Pick another experiment.`; +}; + +/** Create the Intake Evaluation this run publishes under, returning its **name** — + * which is what ``publication.intake.evaluation_id`` takes (the entity id is not it). + * ``experimentId`` conversely is the Experiment's **id**, not its name. */ +export const createRunEvaluation = async ( + workspace: string, + { + experimentId, + experimentName, + filesetName, + signal, + }: { + experimentId: string; + experimentName: string; + filesetName: string; + signal?: AbortSignal; + } +): Promise => { + // Same normalisation as the job name, so the pair reads as one run. + const name = buildEvalJobName(experimentName); + await createEvaluation( + workspace, + { name, experiment_ids: [experimentId], dataset_name: filesetName }, + signal + ); + return name; +}; diff --git a/web/packages/studio/src/components/evaluation/submitEvaluationJob.test.ts b/web/packages/studio/src/components/evaluation/submitEvaluationJob.test.ts index 1368c08844..e973717348 100644 --- a/web/packages/studio/src/components/evaluation/submitEvaluationJob.test.ts +++ b/web/packages/studio/src/components/evaluation/submitEvaluationJob.test.ts @@ -6,6 +6,8 @@ import { buildAgentEvalRequestBody, buildAgentTarget, buildDatasetAgentTarget, + buildDatasetEvalRequestBody, + type DatasetEvalSpec, buildEvalJobName, buildPersistedSpec, injectJudgeModel, @@ -143,6 +145,63 @@ describe('buildAgentEvalRequestBody', () => { expect(body.spec.labels).toBeUndefined(); expect(body.name).toBeUndefined(); }); + + it('names the job after the experiment, falling back to the fileset', () => { + const withExperiment = buildAgentEvalRequestBody(persisted(), { + workspace: 'ws-a', + agent: 'a', + filesetName: 'wise-blue-data', + experimentName: 'wise-blue', + }); + expect(withExperiment.name).toMatch(/^wise-blue-[a-z0-9]{8}$/); + + const filesetOnly = buildAgentEvalRequestBody(persisted(), { + workspace: 'ws-a', + agent: 'a', + filesetName: 'wise-blue-data', + }); + expect(filesetOnly.name).toMatch(/^wise-blue-data-[a-z0-9]{8}$/); + }); + + it('publishes to the named evaluation when one is selected', () => { + const body = buildAgentEvalRequestBody(persisted(), { + workspace: 'ws-a', + agent: 'a', + evaluationId: 'nightly-eval-a3f2', + }); + // agent_name is omitted deliberately: the backend derives it from the agent target. + expect(body.spec.publication).toEqual({ intake: { evaluation_id: 'nightly-eval-a3f2' } }); + }); + + it('omits publication entirely when no evaluation is selected', () => { + const body = buildAgentEvalRequestBody(persisted(), { workspace: 'ws-a', agent: 'a' }); + expect(body.spec.publication).toBeUndefined(); + expect('publication' in body.spec).toBe(false); + }); +}); + +describe('buildDatasetEvalRequestBody', () => { + const datasetConfig: DatasetEvalSpec = { + dataset: [{ prompt: '2+2?' }], + metrics: [metric], + prompt_template: '{{item.prompt}}', + }; + + it('carries publication through the dataset path too, and omits it otherwise', () => { + const withPublication = buildDatasetEvalRequestBody( + datasetConfig, + { workspace: 'ws-a', agent: 'a', evaluationId: 'eval-1' }, + null + ); + expect(withPublication.spec.publication).toEqual({ intake: { evaluation_id: 'eval-1' } }); + + const without = buildDatasetEvalRequestBody( + datasetConfig, + { workspace: 'ws-a', agent: 'a' }, + null + ); + expect(without.spec.publication).toBeUndefined(); + }); }); describe('buildEvalJobName', () => { diff --git a/web/packages/studio/src/components/evaluation/submitEvaluationJob.ts b/web/packages/studio/src/components/evaluation/submitEvaluationJob.ts index a3b13b7d4d..473486cb60 100644 --- a/web/packages/studio/src/components/evaluation/submitEvaluationJob.ts +++ b/web/packages/studio/src/components/evaluation/submitEvaluationJob.ts @@ -9,11 +9,16 @@ import { PLATFORM_BASE_URL } from '@studio/constants/environment'; export const CREATE_NEW = '__create_new__'; export const MODE_DEFAULT = 'default'; -export const MODE_FILESET = 'fileset'; +/** Re-run an existing Experiment, which owns the fileset holding its eval config. */ +export const MODE_EXPERIMENT = 'experiment'; -/** Suggested name for a new eval-config fileset (e.g. "wise-blue"). */ +/** Suggested name for a new experiment (e.g. "wise-blue"). */ export const generateEvalConfigName = (): string => generateDefaultName({ length: 2 }); +/** The fileset that stores an experiment's eval config and data artifacts. */ +export const filesetNameForExperiment = (experimentName: string): string => + `${experimentName}-data`; + /** Default parallelism for a submitted eval (Studio default; the config value is a hint). */ export const DEFAULT_MAX_CONCURRENT_TASKS = 1; @@ -100,8 +105,25 @@ export interface SubmitSelections { agent: string; /** Eval-config fileset name, stored under spec.labels.eval_config_fileset for display. */ filesetName?: string; + /** Experiment this run belongs to; names the job so it reads as one of that experiment's runs. */ + experimentName?: string; + /** Name of an existing Intake Evaluation to publish results under. The job fails if it + * names nothing — the worker never creates it. Omitted means the run publishes nowhere. */ + evaluationId?: string; } +/** ``spec.publication`` for a run that asked to publish, or nothing at all. ``agent_name`` is + * left off deliberately: the backend derives it from the agent target. */ +const publicationSpec = (evaluationId: string | undefined) => + evaluationId ? { publication: { intake: { evaluation_id: evaluationId } } } : {}; + +/** ``{ name }`` for the job, stemmed from the experiment it belongs to and falling back to the + * fileset for a submit that names no experiment. Absent when neither is known. */ +const jobName = (selections: SubmitSelections) => { + const stem = selections.experimentName ?? selections.filesetName; + return stem ? { name: buildEvalJobName(stem) } : {}; +}; + /** Strip an optional ``workspace/`` prefix, returning the bare model/agent name. */ export const bareName = (value: string): string => value.includes('/') ? (value.split('/').pop() ?? value) : value; @@ -171,12 +193,13 @@ export const buildAgentEvalRequestBody = ( spec: PersistedEvalSpec, selections: SubmitSelections ) => ({ - ...(selections.filesetName ? { name: buildEvalJobName(selections.filesetName) } : {}), + ...jobName(selections), spec: { tasks: spec.tasks, target: buildAgentTarget(selections.workspace, selections.agent), max_concurrent_tasks: spec.max_concurrent_tasks ?? DEFAULT_MAX_CONCURRENT_TASKS, ...(selections.filesetName ? { labels: { eval_config_fileset: selections.filesetName } } : {}), + ...publicationSpec(selections.evaluationId), }, }); @@ -195,7 +218,7 @@ export const buildDatasetEvalRequestBody = ( selections: SubmitSelections, judgeModel: string | null ) => ({ - ...(selections.filesetName ? { name: buildEvalJobName(selections.filesetName) } : {}), + ...jobName(selections), spec: { dataset: spec.dataset, metrics: judgeModel @@ -205,6 +228,7 @@ export const buildDatasetEvalRequestBody = ( prompt_template: spec.prompt_template, ...(spec.field_mapping ? { field_mapping: spec.field_mapping } : {}), params: AGENT_RUN_PARAMS, + ...publicationSpec(selections.evaluationId), }, }); diff --git a/web/packages/studio/src/mocks/handlers.ts b/web/packages/studio/src/mocks/handlers.ts index 22c4d7d776..27890818ff 100644 --- a/web/packages/studio/src/mocks/handlers.ts +++ b/web/packages/studio/src/mocks/handlers.ts @@ -24,6 +24,7 @@ import { mockEvaluationSessionsPage, mockEvaluationsPage, mockExperiment, + mockExperimentsPage, } from '@studio/mocks/intake/experiments'; import { createMockAnnotation, @@ -483,6 +484,9 @@ export const handlers = [ const session = mockSessionById(String(params['sessionId'])); return session ? HttpResponse.json(session) : new HttpResponse(null, { status: 404 }); }), + http.get('*/apis/intake/v2/workspaces/:workspace/experiments', () => + HttpResponse.json(mockExperimentsPage()) + ), http.get('*/apis/intake/v2/workspaces/:workspace/experiments/:name', ({ params }) => HttpResponse.json(mockExperiment(String(params['name']))) ), diff --git a/web/packages/studio/src/mocks/intake/experiments.ts b/web/packages/studio/src/mocks/intake/experiments.ts index f688f49e3b..2407766138 100644 --- a/web/packages/studio/src/mocks/intake/experiments.ts +++ b/web/packages/studio/src/mocks/intake/experiments.ts @@ -7,6 +7,7 @@ import type { EvaluationSessionResponse, EvaluationSessionResponsesPage, ExperimentResponse, + ExperimentResponsesPage, } from '@nemo/sdk/generated/platform/schema'; const WORKSPACE = 'default'; @@ -37,6 +38,11 @@ export const mockEvaluationsPage = (): EvaluationResponsesPage => ({ data: MOCK_EVALUATION_NAMES.map(mockEvaluation), }); +/** The group the mock evaluations belong to, so a caller resolving experiment_ids finds a name. */ +export const mockExperimentsPage = (): ExperimentResponsesPage => ({ + data: [{ ...mockExperiment('my-group'), id: 'grp_my-group' }], +}); + const mockRun = ( evaluationName: string, sessionId: string, diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/EvaluationsTab.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/EvaluationsTab.tsx index eec22353b6..420ff9dd60 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/EvaluationsTab.tsx +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/EvaluationsTab.tsx @@ -1,93 +1,49 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; -import { StatusBadge } from '@nemo/common/src/components/StatusBadge'; -import { Button, Flex, Stack, Text } from '@nvidia/foundations-react-core'; -import { - EVAL_JOB_KIND_LABEL, - evalJobDetailRoute, - type EvalJobRow, - hasMixedEvalKinds, -} from '@studio/api/evaluation/utils'; +import { SegmentedControl, Stack } from '@nvidia/foundations-react-core'; +import { EvaluationsTable } from '@studio/routes/agents/AgentDetailRoute/evaluations/EvaluationsTable'; +import { ExperimentsTable } from '@studio/routes/agents/AgentDetailRoute/evaluations/ExperimentsTable'; +import { groupByExperiment } from '@studio/routes/agents/AgentDetailRoute/evaluations/groupByExperiment'; import { DetailPanel } from '@studio/routes/agents/AgentDetailRoute/overview/DetailPanel'; -import { getAgentEvaluationsListRoute } from '@studio/routes/utils'; -import type { FC } from 'react'; -import { Link } from 'react-router'; +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; +import { type FC, useMemo, useState } from 'react'; + +const VIEW_EVALUATIONS = 'evaluations'; +const VIEW_EXPERIMENTS = 'experiments'; + +const VIEW_ITEMS = [ + { value: VIEW_EVALUATIONS, children: 'Evaluations' }, + { value: VIEW_EXPERIMENTS, children: 'Experiments' }, +]; interface EvaluationsTabProps { workspace: string; - evals: EvalJobRow[]; - onRunEvaluation: () => void; + evals: AgentEvaluationRow[]; } -/** Recent evaluation jobs for the agent, linking through to each run. */ -export const EvaluationsTab: FC = ({ workspace, evals, onRunEvaluation }) => { - const showKind = hasMixedEvalKinds(evals); +/** The agent's published evaluations, either flat or rolled up by experiment. Both views read the + * same evaluations; only the grouping differs, which is why this is a toggle and not a tab. */ +export const EvaluationsTab: FC = ({ workspace, evals }) => { + const [view, setView] = useState(VIEW_EVALUATIONS); + const experiments = useMemo(() => groupByExperiment(evals), [evals]); return ( - - Run evaluation - - } - > - {evals.length === 0 ? ( - - No evaluation jobs found for this agent. - - View all evaluations → - - - ) : ( - - {evals.map((job, index) => ( - - 0 ? 'border-t border-base' : ''}`} - > - - - - {job.name} - - {showKind && ( - - ({EVAL_JOB_KIND_LABEL[job.kind]}) - - )} - - {job.configLabel && ( - - Eval Config: {job.configLabel} - - )} - - - - - {job.created_at ? : '—'} - - - - - ))} -
- - View all evaluations → - -
-
- )} + + + + {view === VIEW_EXPERIMENTS ? ( + + ) : ( + + )} + ); }; diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/EvaluationsTable.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/EvaluationsTable.tsx new file mode 100644 index 0000000000..ac93ef0b36 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/EvaluationsTable.tsx @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { Flex, Text } from '@nvidia/foundations-react-core'; +import { + evaluatorScores, + formatCost, + formatLatency, +} from '@studio/routes/agents/AgentDetailRoute/evaluations/formatRollups'; +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; +import { getEvaluationDetailRoute } from '@studio/routes/utils'; +import { FlaskConical } from 'lucide-react'; +import { type ComponentProps, type FC, useCallback } from 'react'; +import { useNavigate } from 'react-router'; + +interface EvaluationsTableProps { + workspace: string; + evaluations: AgentEvaluationRow[]; +} + +/** Every published evaluation for the agent, ungrouped. */ +export const EvaluationsTable: FC = ({ workspace, evaluations }) => { + const navigate = useNavigate(); + const dataViewState = useStudioDataViewState(); + + const makeColumns: ComponentProps>['makeColumns'] = + useCallback( + ({ accessor }) => [ + accessor('name', { + header: 'Evaluation', + cell: ({ row }) => {row.original.name}, + }), + accessor('run_count', { + header: 'Runs', + cell: ({ row }) => {row.original.run_count ?? 0}, + }), + accessor('test_case_count', { + header: 'Test cases', + cell: ({ row }) => {row.original.test_case_count ?? 0}, + }), + accessor('aggregate_scores', { + header: 'Scores', + enableSorting: false, + cell: ({ row }) => { + const scores = evaluatorScores(row.original); + if (scores.length === 0) return ; + // One chip per evaluator, wrapping rather than truncating into an unreadable run-on. + return ( + + {scores.map((score) => ( + + + {score.label} + + {score.value} + + ))} + + ); + }, + }), + accessor('latency_ms', { + header: 'Avg latency', + enableSorting: false, + cell: ({ row }) => {formatLatency(row.original.latency_ms?.mean)}, + }), + accessor('cost_usd', { + header: 'Cost', + enableSorting: false, + cell: ({ row }) => {formatCost(row.original.cost_usd?.sum)}, + }), + accessor('created_at', { + header: 'Created', + cell: ({ row }) => + row.original.created_at ? : '—', + }), + ], + [] + ); + + return ( + + dataViewState={dataViewState} + makeColumns={makeColumns} + // The detail route nests under an experiment; a row without one has nowhere to go. + onRowClick={(row) => + row.experimentName && + navigate(getEvaluationDetailRoute(workspace, row.experimentName, row.name)) + } + attributes={{ + DataViewRoot: { data: evaluations }, + DataViewTableContent: { + renderEmptyState: () => ( + } + header="No published evaluations yet" + emptyMessage="Results appear here once a run finishes and its telemetry is ingested." + /> + ), + }, + }} + /> + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/ExperimentsTable.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/ExperimentsTable.tsx new file mode 100644 index 0000000000..1569d5e36f --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/ExperimentsTable.tsx @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; +import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; +import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; +import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; +import { Text } from '@nvidia/foundations-react-core'; +import type { AgentExperimentRow } from '@studio/routes/agents/AgentDetailRoute/evaluations/groupByExperiment'; +import { getExperimentDetailRoute } from '@studio/routes/utils'; +import { FolderTree } from 'lucide-react'; +import { type ComponentProps, type FC, useCallback } from 'react'; +import { useNavigate } from 'react-router'; + +interface ExperimentsTableProps { + workspace: string; + experiments: AgentExperimentRow[]; +} + +/** The agent's evaluations rolled up by the experiment they belong to. Selecting one opens the + * experiment's own route, which already lists the evaluations under it. */ +export const ExperimentsTable: FC = ({ workspace, experiments }) => { + const navigate = useNavigate(); + const dataViewState = useStudioDataViewState(); + + const makeColumns: ComponentProps>['makeColumns'] = + useCallback( + ({ accessor }) => [ + accessor('name', { + header: 'Experiment', + cell: ({ row }) => {row.original.name}, + }), + accessor('evaluationCount', { + header: 'Evaluations', + cell: ({ row }) => {row.original.evaluationCount}, + }), + accessor('runCount', { + header: 'Runs', + cell: ({ row }) => {row.original.runCount}, + }), + accessor('latestCreatedAt', { + header: 'Latest run', + cell: ({ row }) => + row.original.latestCreatedAt ? ( + + ) : ( + '—' + ), + }), + ], + [] + ); + + return ( + + dataViewState={dataViewState} + makeColumns={makeColumns} + onRowClick={(row) => navigate(getExperimentDetailRoute(workspace, row.name))} + attributes={{ + DataViewRoot: { data: experiments }, + DataViewTableContent: { + renderEmptyState: () => ( + } + header="No experiments yet" + emptyMessage="An experiment appears here once one of its evaluations publishes results for this agent." + /> + ), + }, + }} + /> + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/formatRollups.ts b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/formatRollups.ts new file mode 100644 index 0000000000..0a5ac73494 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/formatRollups.ts @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; + +/** Rollups are computed from ingested telemetry, so every one is absent until a run publishes. */ +export const formatScore = (value: number | null | undefined): string => + typeof value === 'number' ? value.toFixed(2) : '—'; + +export const formatLatency = (ms: number | null | undefined): string => + typeof ms !== 'number' ? '—' : ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`; + +export const formatCost = (usd: number | null | undefined): string => + typeof usd !== 'number' ? '—' : `$${usd < 0.01 ? usd.toFixed(4) : usd.toFixed(2)}`; + +/** Evaluator keys arrive as ``.``, which reads as a stutter whenever the + * score is unnamed and repeats its type (``number-check.number-check``). Keep the part that + * carries meaning: the score name when it adds one, the type otherwise. */ +export const evaluatorLabel = (key: string): string => { + const separator = key.lastIndexOf('.'); + if (separator === -1) return key; + const type = key.slice(0, separator); + const score = key.slice(separator + 1); + return score === type ? type : score; +}; + +export interface EvaluatorScore { + label: string; + value: string; +} + +/** Mean per evaluator. An evaluation names its own evaluators, so these vary row to row and + * cannot each be a column. */ +export const evaluatorScores = (evaluation: AgentEvaluationRow): EvaluatorScore[] => + Object.entries(evaluation.aggregate_scores ?? {}).map(([key, aggregate]) => ({ + label: evaluatorLabel(key), + value: formatScore(aggregate?.mean), + })); diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/groupByExperiment.ts b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/groupByExperiment.ts new file mode 100644 index 0000000000..73c7fa8628 --- /dev/null +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/evaluations/groupByExperiment.ts @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentEvaluationRow } from '@studio/routes/agents/AgentDetailRoute/useAgentDetails'; + +export interface AgentExperimentRow { + name: string; + evaluationCount: number; + runCount: number; + latestCreatedAt: string | null; +} + +/** Roll the agent's evaluations up by experiment. + * + * Derived from the evaluations rather than queried: the experiments endpoint has no + * ``agent_name`` filter, so "which experiments cover this agent" is only answerable through the + * evaluations that name it. An experiment with no published evaluation therefore does not appear. + * Evaluations whose experiment could not be resolved are skipped — there is nothing to group or + * link them under. */ +export const groupByExperiment = (evaluations: AgentEvaluationRow[]): AgentExperimentRow[] => { + const byName = new Map(); + + for (const evaluation of evaluations) { + if (!evaluation.experimentName) continue; + const existing = byName.get(evaluation.experimentName) ?? { + name: evaluation.experimentName, + evaluationCount: 0, + runCount: 0, + latestCreatedAt: null, + }; + existing.evaluationCount += 1; + existing.runCount += evaluation.run_count ?? 0; + if ( + evaluation.created_at && + (!existing.latestCreatedAt || evaluation.created_at > existing.latestCreatedAt) + ) { + existing.latestCreatedAt = evaluation.created_at; + } + byName.set(evaluation.experimentName, existing); + } + + return [...byName.values()].sort((a, b) => + (b.latestCreatedAt ?? '').localeCompare(a.latestCreatedAt ?? '') + ); +}; diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx b/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx index 9c4ab8ca64..1d3590b3ef 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/index.tsx @@ -208,11 +208,7 @@ export const AgentDetailRoute: FC = () => { - setSubmitEvalOpen(true)} - /> + diff --git a/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts b/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts index d77c4a1284..4a8c089ae7 100644 --- a/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts +++ b/web/packages/studio/src/routes/agents/AgentDetailRoute/useAgentDetails.ts @@ -9,12 +9,18 @@ import { useAgentsListAgents, useAgentsListDeployments, } from '@nemo/sdk/generated/agents/api'; -import { fetchEvaluatorJobs } from '@studio/api/evaluation/evaluator-jobs'; -import { targetNameForEvalJob, toEvalJobRow } from '@studio/api/evaluation/utils'; +import { useListEvaluations, useListExperiments } from '@nemo/sdk/generated/platform/api'; +import type { EvaluationResponse } from '@nemo/sdk/generated/platform/schema'; import { RECENT_EVAL_LIMIT } from '@studio/routes/agents/AgentDetailRoute/constants'; -import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { useQueryClient } from '@tanstack/react-query'; import { useMemo } from 'react'; +/** Backend caps page_size at 100. Enough to name the experiments behind a panel's worth of rows. */ +const EXPERIMENT_PAGE_SIZE = 100; + +/** A published evaluation plus the experiment name its detail route is nested under. */ +export type AgentEvaluationRow = EvaluationResponse & { experimentName: string | null }; + interface UseAgentPanelParams { workspace: string; agentName?: string; @@ -58,19 +64,19 @@ export const useAgentDetails = ({ const agentsData = agentsResponse?.data; const deploymentsData = deploymentsResponse?.data; - // Recent evaluations targeting this agent. The platform's job filter API - // doesn't expose ``spec.agent`` as a top-level filter, so we fetch the - // workspace's eval jobs and filter client-side. Capped at the most recent - // N to keep the panel scannable; the full list is on the evaluations route. - const { data: agentEvalsData } = useQuery({ - queryKey: ['evaluator-jobs', workspace, 'panel', agentName] as const, - queryFn: ({ signal }) => - fetchEvaluatorJobs(workspace, signal, (all) => { - const matched = all.filter((j) => targetNameForEvalJob(j) === agentName).length; - return matched >= RECENT_EVAL_LIMIT; - }), - enabled: !!agentName && !!workspace, - }); + // Recent evaluations for this agent, read from Intake rather than the job list: only Intake + // carries the telemetry rollups (latency, cost, tokens, evaluator scores) the panel shows, and + // ``filter[agent_name]`` scopes them server-side. An evaluation appears once its run publishes, + // so in-flight and failed runs are not here — they stay on the workspace-wide evaluations route. + const { data: agentEvalsResponse } = useListEvaluations( + workspace, + { + filter: { agent_name: agentName ?? '' }, + page_size: RECENT_EVAL_LIMIT, + sort: '-created_at', + }, + { query: { enabled: !!agentName && !!workspace } } + ); const deleteDeploymentMutation = useAgentsDeleteDeployment({ mutation: { @@ -91,13 +97,28 @@ export const useAgentDetails = ({ [deploymentsData, agentName] ); - const agentEvals = useMemo(() => { + // The evaluation detail route is nested under an experiment, but an evaluation carries only + // ``experiment_ids``. One list call resolves them; the panel shows a handful of rows, so this + // is cheaper than a lookup per row. + const { data: experimentsResponse } = useListExperiments( + workspace, + { page_size: EXPERIMENT_PAGE_SIZE }, + { query: { enabled: !!agentName && !!workspace } } + ); + + const agentEvals: AgentEvaluationRow[] = useMemo(() => { if (!agentName) return []; - const all = (agentEvalsData ?? []).map(toEvalJobRow); - // Match either the bare agent name or a workspace-prefixed ref. - const matches = all.filter((job) => job.agentName === agentName); - return matches.slice(0, RECENT_EVAL_LIMIT); - }, [agentEvalsData, agentName]); + const namesById = new Map( + (experimentsResponse?.data ?? []).map((experiment) => [experiment.id, experiment.name]) + ); + return (agentEvalsResponse?.data ?? []).map((evaluation) => ({ + ...evaluation, + // First id, mirroring the API's own deprecated ``experiment_group_id`` ("first of + // experiment_ids"). Null when the experiment is missing, which drops the row's link + // rather than routing somewhere broken. + experimentName: namesById.get(evaluation.experiment_ids[0] ?? '') ?? null, + })); + }, [agentEvalsResponse, experimentsResponse, agentName]); const healthyDeployments = useMemo( () => agentDeployments.filter((d) => d.status === 'running'),