diff --git a/integ-tests/create-edge-cases.test.ts b/integ-tests/create-edge-cases.test.ts index 93849328c..d7982eed8 100644 --- a/integ-tests/create-edge-cases.test.ts +++ b/integ-tests/create-edge-cases.test.ts @@ -39,6 +39,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', exit_reason: 'failure', error_name: 'ValidationError', error_source: 'user', + agent_environment: 'runtime', agent_language: 'python', has_agent: 'true', }); @@ -143,6 +144,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', telemetry.assertMetricEmitted({ command: 'create', exit_reason: 'success', + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', @@ -194,3 +196,38 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', }); }); }); + +const isPreviewBuild = process.env.BUILD_PREVIEW === '1'; + +describe.skipIf(!isPreviewBuild || !prereqs.npm || !prereqs.git)('integration: create harness project', () => { + let testDir: string; + const telemetry = createTelemetryHelper(); + + beforeAll(async () => { + testDir = join(tmpdir(), `agentcore-integ-create-harness-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + }); + + afterAll(async () => { + telemetry.destroy(); + await rm(testDir, { recursive: true, force: true }); + }); + + it('creates a harness project with defaults', async () => { + const name = `Hrn${Date.now().toString().slice(-6)}`; + const result = await runCLI(['create', '--name', name, '--model-provider', 'bedrock', '--json'], testDir, { + env: telemetry.env, + }); + + expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + telemetry.assertMetricEmitted({ + command: 'create', + exit_reason: 'success', + agent_environment: 'harness', + has_agent: 'true', + }); + }); +}); diff --git a/integ-tests/create-frameworks.test.ts b/integ-tests/create-frameworks.test.ts index 2fcce842b..410d137b2 100644 --- a/integ-tests/create-frameworks.test.ts +++ b/integ-tests/create-frameworks.test.ts @@ -69,6 +69,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen telemetry.assertMetricEmitted({ command: 'create', exit_reason: 'success', + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'langchain_langgraph', model_provider: 'bedrock', diff --git a/integ-tests/dev-server.test.ts b/integ-tests/dev-server.test.ts index f2babe662..0a7fea149 100644 --- a/integ-tests/dev-server.test.ts +++ b/integ-tests/dev-server.test.ts @@ -109,6 +109,7 @@ describe('integration: dev server', () => { command: 'dev', dev_action: 'server', ui_mode: 'terminal', + agent_environment: 'runtime', exit_reason: 'success', }); telemetry.clearEntries(); @@ -123,6 +124,7 @@ describe('integration: dev server', () => { command: 'dev', dev_action: 'invoke', ui_mode: 'terminal', + agent_environment: 'runtime', exit_reason: 'success', agent_protocol: 'http', }); @@ -135,6 +137,7 @@ describe('integration: dev server', () => { telemetry.assertMetricEmitted({ command: 'dev', dev_action: 'invoke', + agent_environment: 'runtime', exit_reason: 'failure', }); @@ -162,9 +165,51 @@ describe('integration: dev server', () => { telemetry.assertMetricEmitted({ command: 'dev', dev_action: 'server', + agent_environment: 'runtime', exit_reason: 'failure', }); }, 15000 ); }); + +const isPreviewBuild = process.env.BUILD_PREVIEW === '1'; + +describe.skipIf(!isPreviewBuild || !hasNpm || !hasGit || !hasUv)('integration: dev with harness-only project', () => { + const telemetry = createTelemetryHelper(); + let projectPath: string; + + beforeAll(async () => { + const dir = join(tmpdir(), `agentcore-dev-harness-${Date.now()}`); + await mkdir(dir, { recursive: true }); + + // Create a harness-only project + const createResult = await runCLI( + ['create', '--name', 'DevHarness', '--model-provider', 'bedrock', '--json'], + dir, + { env: telemetry.env } + ); + const json = JSON.parse(createResult.stdout); + projectPath = json.projectPath; + }); + + afterAll(async () => { + telemetry.destroy(); + if (projectPath) await rm(projectPath, { recursive: true, force: true }); + }); + + // This test currently fails due to https://github.com/aws/agentcore-cli/issues/1406 + it.skip('dev --logs on harness-only project should fail with validation error', async () => { + telemetry.clearEntries(); + const result = await runCLI(['dev', '--logs', '--skip-deploy'], projectPath, { env: telemetry.env }); + + expect(result.exitCode).toBe(1); + + telemetry.assertMetricEmitted({ + command: 'dev', + dev_action: 'server', + agent_environment: 'harness', + exit_reason: 'failure', + }); + }); +}); diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index 78e566779..aa3f8e84e 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -9,10 +9,11 @@ import type { TargetLanguage, } from '../../../schema'; import { LIFECYCLE_TIMEOUT_MAX, LIFECYCLE_TIMEOUT_MIN } from '../../../schema'; +import { ANSI } from '../../constants'; import { getErrorMessage } from '../../errors'; import { isPreviewEnabled } from '../../feature-flags'; import { harnessPrimitive } from '../../primitives/registry'; -import { runCliCommand } from '../../telemetry/cli-command-run.js'; +import { runCliCommand, withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { AgentFramework, AgentLanguage, @@ -139,78 +140,83 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { const name = options.name ?? options.projectName; const projectName = options.projectName ?? name; - const validation = validateCreateHarnessOptions( + const result = await withCommandRunTelemetry( + 'create', { - name, - projectName, - modelProvider: options.modelProvider, - modelId: options.modelId, - apiKeyArn: options.apiKeyArn, + agent_environment: 'harness' as const, + has_agent: true, + model_provider: standardize(ModelProviderEnum, options.modelProvider ?? 'bedrock'), + memory_type: standardize(MemoryType, options.harnessMemory === false ? 'none' : 'longandshortterm'), + network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'), }, - cwd - ); - if (!validation.valid) { - if (options.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); - } else { - console.error(validation.error); - } - process.exit(1); - } + async () => { + const validation = validateCreateHarnessOptions( + { + name, + projectName, + modelProvider: options.modelProvider, + modelId: options.modelId, + apiKeyArn: options.apiKeyArn, + }, + cwd + ); + if (!validation.valid) { + return { success: false as const, error: new ValidationError(validation.error!) }; + } - // Progress callback - const green = '\x1b[32m'; - const reset = '\x1b[0m'; - const onProgress: ProgressCallback | undefined = options.json - ? undefined - : (step, status) => { - if (status === 'done') console.log(`${green}[done]${reset} ${step}`); - else if (status === 'error') console.log(`\x1b[31m[error]${reset} ${step}`); - }; + // Progress callback + const onProgress: ProgressCallback | undefined = options.json + ? undefined + : (step, status) => { + if (status === 'done') console.log(`${ANSI.green}[done]${ANSI.reset} ${step}`); + else if (status === 'error') console.log(`${ANSI.red}[error]${ANSI.reset} ${step}`); + }; - const provider = ( - options.modelProvider ? normalizeHarnessModelProvider(options.modelProvider) : 'bedrock' - ) as HarnessModelProvider; - const defaultModelIds: Record = { - bedrock: 'global.anthropic.claude-sonnet-4-6', - open_ai: 'gpt-5', - gemini: 'gemini-2.5-flash', - }; - const modelId = options.modelId ?? defaultModelIds[provider] ?? 'global.anthropic.claude-sonnet-4-6'; - - const containerOption = harnessPrimitive!.parseContainerFlag(options.container); - - const result = await createProjectWithHarness({ - name: name!, - projectName: projectName!, - cwd, - modelProvider: provider, - modelId, - apiKeyArn: options.apiKeyArn, - containerUri: containerOption.containerUri, - dockerfilePath: containerOption.dockerfilePath, - skipMemory: options.harnessMemory === false, - maxIterations: options.maxIterations ? Number(options.maxIterations) : undefined, - maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined, - timeoutSeconds: options.timeout ? Number(options.timeout) : undefined, - truncationStrategy: options.truncationStrategy as 'sliding_window' | 'summarization' | undefined, - networkMode: options.networkMode as NetworkMode | undefined, - subnets: parseCommaSeparatedList(options.subnets), - securityGroups: parseCommaSeparatedList(options.securityGroups), - idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined, - maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined, - sessionStoragePath: options.sessionStorageMountPath, - skipGit: options.skipGit, - skipInstall: options.skipInstall, - onProgress, - }); + const provider = ( + options.modelProvider ? normalizeHarnessModelProvider(options.modelProvider) : 'bedrock' + ) as HarnessModelProvider; + const defaultModelIds: Record = { + bedrock: 'global.anthropic.claude-sonnet-4-6', + open_ai: 'gpt-5', + gemini: 'gemini-2.5-flash', + }; + const modelId = options.modelId ?? defaultModelIds[provider] ?? 'global.anthropic.claude-sonnet-4-6'; + + const containerOption = harnessPrimitive!.parseContainerFlag(options.container); + + return createProjectWithHarness({ + name: name!, + projectName: projectName!, + cwd, + modelProvider: provider, + modelId, + apiKeyArn: options.apiKeyArn, + containerUri: containerOption.containerUri, + dockerfilePath: containerOption.dockerfilePath, + skipMemory: options.harnessMemory === false, + maxIterations: options.maxIterations ? Number(options.maxIterations) : undefined, + maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined, + timeoutSeconds: options.timeout ? Number(options.timeout) : undefined, + truncationStrategy: options.truncationStrategy as 'sliding_window' | 'summarization' | undefined, + networkMode: options.networkMode as NetworkMode | undefined, + subnets: parseCommaSeparatedList(options.subnets), + securityGroups: parseCommaSeparatedList(options.securityGroups), + idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined, + maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined, + sessionStoragePath: options.sessionStorageMountPath, + skipGit: options.skipGit, + skipInstall: options.skipInstall, + onProgress, + }); + } + ); if (options.json) { - console.log(JSON.stringify(result)); + console.log(JSON.stringify(serializeResult(result))); } else if (result.success) { printCreateHarnessSummary(projectName!, name!); } else { - console.error(result.error); + console.error(result.error instanceof Error ? result.error.message : 'Create failed'); } process.exit(result.success ? 0 : 1); } @@ -245,6 +251,7 @@ async function handleCreateCLI(options: CreateOptions): Promise { } const knownAttrs = { + agent_environment: 'runtime' as const, agent_language: standardize(AgentLanguage, options.language), agent_framework: standardize(AgentFramework, options.framework), model_provider: standardize(ModelProviderEnum, options.modelProvider), diff --git a/src/cli/commands/deploy/__tests__/utils.test.ts b/src/cli/commands/deploy/__tests__/utils.test.ts index 2ba22c15a..256eed986 100644 --- a/src/cli/commands/deploy/__tests__/utils.test.ts +++ b/src/cli/commands/deploy/__tests__/utils.test.ts @@ -16,6 +16,7 @@ describe('computeDeployAttrs', () => { expect(computeDeployAttrs(projectSpec, 'diff')).toEqual({ runtime_count: 2, + harness_count: 0, memory_count: 1, credential_count: 3, evaluator_count: 1, @@ -31,6 +32,7 @@ describe('computeDeployAttrs', () => { it('returns zeros for empty spec', () => { expect(computeDeployAttrs({}, 'deploy')).toEqual({ runtime_count: 0, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, diff --git a/src/cli/commands/deploy/utils.ts b/src/cli/commands/deploy/utils.ts index c866ded9e..d6362e114 100644 --- a/src/cli/commands/deploy/utils.ts +++ b/src/cli/commands/deploy/utils.ts @@ -3,6 +3,7 @@ import type { DeployMode } from '../../telemetry/schemas/common-shapes'; export const DEFAULT_DEPLOY_ATTRS = { runtime_count: 0, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, @@ -19,6 +20,7 @@ export function computeDeployAttrs(projectSpec: Partial, m const policyEngines = projectSpec.policyEngines ?? []; return { runtime_count: (projectSpec.runtimes ?? []).length, + harness_count: (projectSpec.harnesses ?? []).length, memory_count: (projectSpec.memories ?? []).length, credential_count: (projectSpec.credentials ?? []).length, evaluator_count: (projectSpec.evaluators ?? []).length, diff --git a/src/cli/commands/dev/command.tsx b/src/cli/commands/dev/command.tsx index b935692f3..d60624940 100644 --- a/src/cli/commands/dev/command.tsx +++ b/src/cli/commands/dev/command.tsx @@ -201,6 +201,7 @@ export const registerDev = (program: Command) => { const execResult = await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime' as const, dev_action: 'exec' as const, ui_mode: 'terminal' as const, has_stream: false, @@ -239,6 +240,7 @@ export const registerDev = (program: Command) => { const invokeResult = await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime' as const, dev_action: 'invoke' as const, ui_mode: 'terminal' as const, has_stream: opts.stream ?? false, @@ -301,6 +303,7 @@ export const registerDev = (program: Command) => { const serverResult = await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime' as const, dev_action: 'server' as const, ui_mode: 'terminal' as const, has_stream: false, @@ -353,6 +356,7 @@ export const registerDev = (program: Command) => { if (opts.logs) { // Preview: harness-only projects need deploy then print invoke instructions if (isPreviewEnabled() && supportedAgents.length === 0 && hasHarnesses) { + recorder.set({ agent_environment: 'harness' as const }); if (!opts.skipDeploy) { await runCliDeploy(); } diff --git a/src/cli/commands/invoke/__tests__/utils.test.ts b/src/cli/commands/invoke/__tests__/utils.test.ts new file mode 100644 index 000000000..7187c7409 --- /dev/null +++ b/src/cli/commands/invoke/__tests__/utils.test.ts @@ -0,0 +1,100 @@ +import { computeInvokeAttrs } from '../utils'; +import { describe, expect, it } from 'vitest'; + +describe('computeInvokeAttrs', () => { + it('returns runtime when preview is false regardless of harness flags', () => { + const attrs = computeInvokeAttrs({ + preview: false, + harnessName: 'my-harness', + harnessCount: 1, + runtimeCount: 0, + stream: true, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('runtime'); + expect(attrs.agent_protocol).toBe('http'); + }); + + it('returns harness when harnessName is set and preview is true', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessName: 'my-harness', + harnessCount: 1, + runtimeCount: 1, + stream: false, + hasSessionId: true, + }); + expect(attrs.agent_environment).toBe('harness'); + expect(attrs.agent_protocol).toBeUndefined(); + expect(attrs.has_session_id).toBe(true); + }); + + it('returns harness when harnessArn is set and preview is true', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessArn: 'arn:aws:bedrock:us-east-1:123:harness/h1', + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('harness'); + expect(attrs.agent_protocol).toBeUndefined(); + }); + + it('returns harness when project has only harnesses', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessCount: 2, + runtimeCount: 0, + stream: false, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('harness'); + }); + + it('returns runtime for mixed project without explicit harness flag', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessCount: 1, + runtimeCount: 1, + stream: false, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('runtime'); + expect(attrs.agent_protocol).toBe('http'); + }); + + it('passes auth_type based on bearerToken', () => { + const withToken = computeInvokeAttrs({ + preview: false, + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + bearerToken: 'tok', + }); + expect(withToken.auth_type).toBe('bearer_token'); + + const withoutToken = computeInvokeAttrs({ + preview: false, + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + }); + expect(withoutToken.auth_type).toBe('sigv4'); + }); + + it('uses provided agentProtocol for runtime', () => { + const attrs = computeInvokeAttrs({ + preview: false, + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + agentProtocol: 'MCP', + }); + expect(attrs.agent_protocol).toBe('mcp'); + }); +}); diff --git a/src/cli/commands/invoke/command.tsx b/src/cli/commands/invoke/command.tsx index 0ef293d9a..4817a72b9 100644 --- a/src/cli/commands/invoke/command.tsx +++ b/src/cli/commands/invoke/command.tsx @@ -2,7 +2,6 @@ import { ValidationError, serializeResult } from '../../../lib'; import { getErrorMessage } from '../../errors'; import { isPreviewEnabled } from '../../feature-flags'; import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; -import { AgentProtocol, AuthType, standardize } from '../../telemetry/schemas/common-shapes.js'; import { renderTUI } from '../../tui'; import { COMMAND_DESCRIPTIONS } from '../../tui/copy'; import { requireProject, requireTTY } from '../../tui/guards'; @@ -10,6 +9,7 @@ import { parseHeaderFlags } from '../shared/header-utils'; import { type InvokeContext, handleHarnessInvokeByArn, handleInvoke, loadInvokeConfig } from './action'; import { resolvePrompt } from './resolve-prompt'; import type { InvokeOptions, InvokeResult } from './types'; +import { computeInvokeAttrs } from './utils'; import { validateInvokeOptions } from './validate'; import type { Command } from '@commander-js/extra-typings'; import { Text, render } from 'ink'; @@ -30,12 +30,6 @@ function stopSpinner(spinner: NodeJS.Timeout): void { process.stderr.write('\r\x1b[K'); // Clear line } -function resolveProtocol(options: InvokeOptions, projectProtocol?: string): string { - if (projectProtocol) return projectProtocol.toLowerCase(); - if (options.tool) return 'mcp'; - return 'http'; -} - async function handleInvokeCLI(options: InvokeOptions, preloadedContext?: InvokeContext): Promise { const validation = validateInvokeOptions(options); if (!validation.valid) { @@ -263,15 +257,17 @@ export const registerInvoke = (program: Command) => { ) { const result = await withCommandRunTelemetry( 'invoke', - { - has_stream: cliOptions.stream ?? false, - has_session_id: !!cliOptions.sessionId, - auth_type: standardize(AuthType, cliOptions.bearerToken ? 'bearer_token' : 'sigv4'), - agent_protocol: standardize( - AgentProtocol, - resolveProtocol({ tool: cliOptions.tool } as InvokeOptions, agentProtocol) - ), - }, + computeInvokeAttrs({ + preview: isPreviewEnabled(), + harnessName: cliOptions.harness, + harnessArn: cliOptions.harnessArn, + harnessCount: invokeContext?.project.harnesses?.length ?? 0, + runtimeCount: invokeContext?.project.runtimes.length ?? 0, + stream: cliOptions.stream ?? false, + hasSessionId: !!cliOptions.sessionId, + bearerToken: cliOptions.bearerToken, + agentProtocol: agentProtocol ?? (cliOptions.tool ? 'mcp' : undefined), + }), async (): Promise => { if (!resolved.success) { return { success: false, error: new ValidationError(resolved.error ?? 'Prompt resolution failed') }; diff --git a/src/cli/commands/invoke/utils.ts b/src/cli/commands/invoke/utils.ts new file mode 100644 index 000000000..beda64134 --- /dev/null +++ b/src/cli/commands/invoke/utils.ts @@ -0,0 +1,33 @@ +import { AgentEnvironment, AgentProtocol, AuthType, standardize } from '../../telemetry/schemas/common-shapes.js'; + +function isHarnessInvoke(options: { + harnessName?: string; + harnessArn?: string; + harnessCount: number; + runtimeCount: number; +}): boolean { + if (options.harnessName || options.harnessArn) return true; + if (options.harnessCount > 0 && options.runtimeCount === 0) return true; + return false; +} + +export function computeInvokeAttrs(options: { + preview: boolean; + harnessName?: string; + harnessArn?: string; + harnessCount: number; + runtimeCount: number; + stream: boolean; + hasSessionId: boolean; + bearerToken?: string; + agentProtocol?: string; +}) { + const isHarness = options.preview && isHarnessInvoke(options); + return { + agent_environment: standardize(AgentEnvironment, isHarness ? 'harness' : 'runtime'), + has_stream: options.stream, + has_session_id: options.hasSessionId, + auth_type: standardize(AuthType, options.bearerToken ? 'bearer_token' : 'sigv4'), + agent_protocol: isHarness ? undefined : standardize(AgentProtocol, options.agentProtocol ?? 'http'), + }; +} diff --git a/src/cli/telemetry/__tests__/client.test.ts b/src/cli/telemetry/__tests__/client.test.ts index 84946cad2..72f95f482 100644 --- a/src/cli/telemetry/__tests__/client.test.ts +++ b/src/cli/telemetry/__tests__/client.test.ts @@ -90,6 +90,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'create', { + agent_environment: 'runtime', agent_language: 'rust' as never, agent_framework: 'strands', model_provider: 'bedrock', @@ -112,6 +113,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'create', { + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', @@ -172,6 +174,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -204,6 +207,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -229,6 +233,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -249,6 +254,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -277,6 +283,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, diff --git a/src/cli/telemetry/schemas/__tests__/command-run.test.ts b/src/cli/telemetry/schemas/__tests__/command-run.test.ts index 65b42c74b..5115ea1d8 100644 --- a/src/cli/telemetry/schemas/__tests__/command-run.test.ts +++ b/src/cli/telemetry/schemas/__tests__/command-run.test.ts @@ -39,6 +39,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid deploy attrs', () => { const attrs = { runtime_count: 2, + harness_count: 1, memory_count: 1, credential_count: 0, evaluator_count: 0, @@ -56,6 +57,7 @@ describe('COMMAND_SCHEMAS', () => { expect(() => COMMAND_SCHEMAS.deploy.parse({ runtime_count: -1, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, @@ -73,6 +75,7 @@ describe('COMMAND_SCHEMAS', () => { expect(() => COMMAND_SCHEMAS.deploy.parse({ runtime_count: 1.5, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, @@ -88,6 +91,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid create attrs', () => { const attrs = { + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', @@ -104,6 +108,7 @@ describe('COMMAND_SCHEMAS', () => { it('rejects create attrs with invalid enum value', () => { expect(() => COMMAND_SCHEMAS.create.parse({ + agent_environment: 'runtime', agent_language: 'rust', agent_framework: 'strands', model_provider: 'bedrock', @@ -132,6 +137,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid dev invoke attrs', () => { const attrs = { + agent_environment: 'runtime', dev_action: 'invoke', ui_mode: 'terminal', has_stream: true, @@ -143,6 +149,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid dev server browser attrs', () => { const attrs = { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'browser', has_stream: false, @@ -154,6 +161,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts dev exec attrs', () => { const attrs = { + agent_environment: 'runtime', dev_action: 'exec', ui_mode: 'terminal', has_stream: false, @@ -166,6 +174,7 @@ describe('COMMAND_SCHEMAS', () => { it('rejects dev attrs with invalid action', () => { expect(() => COMMAND_SCHEMAS.dev.parse({ + agent_environment: 'runtime', dev_action: 'unknown', ui_mode: 'terminal', has_stream: false, @@ -178,6 +187,7 @@ describe('COMMAND_SCHEMAS', () => { it('rejects dev attrs with invalid ui_mode', () => { expect(() => COMMAND_SCHEMAS.dev.parse({ + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'headless', has_stream: false, @@ -216,11 +226,12 @@ describe('type safety', () => { it('no command schema contains arbitrary string fields', () => { for (const [cmd, schema] of Object.entries(COMMAND_SCHEMAS)) { for (const [field, zodType] of Object.entries(schema.shape)) { + const inner = zodType instanceof z.ZodOptional ? zodType.unwrap() : zodType; const safe = - zodType instanceof z.ZodEnum || - zodType instanceof z.ZodBoolean || - zodType instanceof z.ZodNumber || - zodType instanceof z.ZodLiteral; + inner instanceof z.ZodEnum || + inner instanceof z.ZodBoolean || + inner instanceof z.ZodNumber || + inner instanceof z.ZodLiteral; expect(safe, `${cmd}.${field} is an unsafe type`).toBe(true); } } @@ -240,6 +251,7 @@ describe('type safety', () => { describe('resilientParse', () => { it('passes valid attrs through unchanged', () => { const attrs = { + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', @@ -273,14 +285,14 @@ describe('resilientParse', () => { it('defaults missing required fields to unknown', () => { const result = resilientParse(COMMAND_SCHEMAS.create, { agent_language: 'python' }); expect(result.agent_language).toBe('python'); - expect(result.agent_framework).toBe('unknown'); - expect(result.model_provider).toBe('unknown'); + expect(result.agent_environment).toBe('unknown'); + expect(result.has_agent).toBe('unknown'); }); it('defaults all fields to unknown when all are invalid', () => { const result = resilientParse(COMMAND_SCHEMAS.create, {}); for (const value of Object.values(result)) { - expect(value).toBe('unknown'); + expect(value === 'unknown' || value === undefined).toBe(true); } }); diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index 4bb25bd3a..eaf5a9c17 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -1,4 +1,5 @@ import { + AgentEnvironment, AgentFramework, AgentLanguage, AgentProtocol, @@ -33,13 +34,14 @@ import { import { z } from 'zod'; const CreateAttrs = safeSchema({ - agent_language: AgentLanguage, - agent_framework: AgentFramework, + agent_environment: AgentEnvironment, + agent_language: AgentLanguage.optional(), + agent_framework: AgentFramework.optional(), model_provider: ModelProvider, memory_type: MemoryType, - agent_protocol: AgentProtocol, - build_type: BuildType, - agent_source: AgentSource, + agent_protocol: AgentProtocol.optional(), + build_type: BuildType.optional(), + agent_source: AgentSource.optional(), network_mode: NetworkMode, has_agent: z.boolean(), }); @@ -95,6 +97,7 @@ const AddPolicyAttrs = safeSchema({ const DeployAttrs = safeSchema({ runtime_count: Count, + harness_count: Count, memory_count: Count, credential_count: Count, evaluator_count: Count, @@ -107,18 +110,20 @@ const DeployAttrs = safeSchema({ }); const DevAttrs = safeSchema({ + agent_environment: AgentEnvironment, dev_action: DevAction, ui_mode: UiMode, has_stream: z.boolean(), - agent_protocol: AgentProtocol, + agent_protocol: AgentProtocol.optional(), invoke_count: Count, }); const InvokeAttrs = safeSchema({ + agent_environment: AgentEnvironment, has_stream: z.boolean(), has_session_id: z.boolean(), auth_type: AuthType, - agent_protocol: AgentProtocol, + agent_protocol: AgentProtocol.optional(), }); const StatusAttrs = safeSchema({ filter_type: FilterType, filter_state: FilterState }); diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index e16230098..475b0100b 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -3,7 +3,8 @@ import { z } from 'zod'; // Type-safe schema builder: rejects z.string() at compile time. // Only z.enum(), z.boolean(), z.number(), and z.literal() are allowed as field types. // eslint-disable-next-line @typescript-eslint/no-explicit-any -type SafeField = z.ZodEnum | z.ZodBoolean | z.ZodNumber | z.ZodLiteral; +type BaseSafeField = z.ZodEnum | z.ZodBoolean | z.ZodNumber | z.ZodLiteral; +type SafeField = BaseSafeField | z.ZodOptional; export function safeSchema>(shape: T) { return z.object(shape); } @@ -71,6 +72,7 @@ export const FilterType = z.enum([ 'harness', 'none', ]); +export const AgentEnvironment = z.enum(['harness', 'runtime']); export const AgentFramework = z.enum(['strands', 'langchain_langgraph', 'googleadk', 'openaiagents']); export const GatewayTargetHost = z.enum(['lambda', 'agentcoreruntime']); export const GatewayTargetType = z.enum([ @@ -153,6 +155,7 @@ export type DeployMode = z.infer; Keys are the field names as they appear in emitted metrics. */ export const ATTRIBUTES = { + agent_environment: AgentEnvironment, dev_action: DevAction, agent_source: AgentSource, attach_gateway_count: Count, diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts index 645bcc095..029f80c66 100644 --- a/src/cli/tui/screens/create/useCreateFlow.ts +++ b/src/cli/tui/screens/create/useCreateFlow.ts @@ -26,6 +26,7 @@ import { credentialPrimitive } from '../../../primitives/registry'; import { createDefaultProjectSpec } from '../../../project'; import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { + AgentEnvironment, AgentFramework, AgentLanguage, AgentProtocol, @@ -259,16 +260,32 @@ export function useCreateFlow(cwd: string): CreateFlowState { useEffect(() => { if (phase !== 'running') return; + const isHarness = addHarnessConfig !== null; const attrs = { - agent_language: standardize(AgentLanguage, addAgentConfig?.language ?? 'Python'), - agent_framework: standardize(AgentFramework, addAgentConfig?.framework), - model_provider: standardize(ModelProvider, addAgentConfig?.modelProvider), - memory_type: standardize(MemoryEnum, addAgentConfig?.memory ?? 'none'), - agent_protocol: standardize(AgentProtocol, addAgentConfig?.protocol ?? 'HTTP'), - build_type: standardize(BuildType, addAgentConfig?.buildType ?? 'CodeZip'), - agent_source: standardize(AgentSource, addAgentConfig?.agentType ?? 'create'), - network_mode: standardize(NetworkMode, addAgentConfig?.networkMode ?? 'PUBLIC'), - has_agent: addAgentConfig !== null, + agent_environment: standardize(AgentEnvironment, isHarness ? 'harness' : 'runtime'), + // true when either an agent or harness config is set (non-null/non-undefined) + has_agent: Boolean(addAgentConfig) || Boolean(addHarnessConfig), + model_provider: standardize( + ModelProvider, + isHarness ? addHarnessConfig?.modelProvider : addAgentConfig?.modelProvider + ), + memory_type: standardize( + MemoryEnum, + isHarness ? (addHarnessConfig?.skipMemory ? 'none' : 'longandshortterm') : (addAgentConfig?.memory ?? 'none') + ), + build_type: isHarness ? undefined : standardize(BuildType, addAgentConfig?.buildType ?? 'CodeZip'), + network_mode: standardize( + NetworkMode, + isHarness ? (addHarnessConfig?.networkMode ?? 'PUBLIC') : (addAgentConfig?.networkMode ?? 'PUBLIC') + ), + ...(isHarness + ? {} + : { + agent_language: standardize(AgentLanguage, addAgentConfig?.language ?? 'Python'), + agent_framework: standardize(AgentFramework, addAgentConfig?.framework), + agent_protocol: standardize(AgentProtocol, addAgentConfig?.protocol ?? 'HTTP'), + agent_type: standardize(AgentSource, addAgentConfig?.agentType ?? 'create'), + }), }; const run = async (): Promise<{ success: true } | { success: false; error: Error }> => { diff --git a/src/cli/tui/screens/invoke/useInvokeFlow.ts b/src/cli/tui/screens/invoke/useInvokeFlow.ts index a770cc7ae..44481a39d 100644 --- a/src/cli/tui/screens/invoke/useInvokeFlow.ts +++ b/src/cli/tui/screens/invoke/useInvokeFlow.ts @@ -22,6 +22,7 @@ import { mcpListTools, } from '../../../aws'; import { invokeHarness } from '../../../aws/agentcore-harness'; +import { computeInvokeAttrs } from '../../../commands/invoke/utils'; import { ANSI } from '../../../constants'; import { getErrorMessage } from '../../../errors'; import { isPreviewEnabled } from '../../../feature-flags'; @@ -35,7 +36,6 @@ import { } from '../../../operations/fetch-access'; import { generateSessionId } from '../../../operations/session'; import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; -import { AgentProtocol, AuthType, standardize } from '../../../telemetry/schemas/common-shapes.js'; import { useCallback, useEffect, useRef, useState } from 'react'; /** Structured message part for rich AGUI event rendering */ @@ -138,12 +138,16 @@ export function useInvokeFlow(options: InvokeFlowOptions = {}): InvokeFlowState const result = await withCommandRunTelemetry( 'invoke', - { - has_stream: true, - has_session_id: !!initialSessionId, - auth_type: standardize(AuthType, initialBearerToken ? 'bearer_token' : 'sigv4'), - agent_protocol: standardize(AgentProtocol, firstProtocol), - }, + computeInvokeAttrs({ + preview: isPreviewEnabled(), + harnessName: initialHarnessName, + harnessCount: project?.harnesses?.length ?? 0, + runtimeCount: project?.runtimes?.length ?? 0, + stream: true, + hasSessionId: !!initialSessionId, + bearerToken: initialBearerToken, + agentProtocol: firstProtocol, + }), async () => { if (!project) { return { success: false as const, error: new ResourceNotFoundError('No agentcore project found.') };