From af151625ca50718a77f32834a5f91491667d10d5 Mon Sep 17 00:00:00 2001 From: hkobew Date: Wed, 27 May 2026 23:00:27 -0400 Subject: [PATCH 01/10] feat(telemetry): support agent_environment for runtime/harness distinction --- src/cli/commands/create/command.tsx | 1 + .../commands/deploy/__tests__/utils.test.ts | 2 ++ src/cli/commands/deploy/utils.ts | 2 ++ src/cli/commands/dev/command.tsx | 4 +++ src/cli/commands/invoke/command.tsx | 28 +++++++--------- src/cli/commands/invoke/utils.ts | 33 +++++++++++++++++++ src/cli/telemetry/__tests__/client.test.ts | 7 ++++ .../schemas/__tests__/command-run.test.ts | 26 +++++++++++---- src/cli/telemetry/schemas/command-run.ts | 25 ++++++++------ src/cli/telemetry/schemas/common-shapes.ts | 5 ++- src/cli/tui/screens/create/useCreateFlow.ts | 2 ++ src/cli/tui/screens/invoke/useInvokeFlow.ts | 18 ++++++---- 12 files changed, 112 insertions(+), 41 deletions(-) create mode 100644 src/cli/commands/invoke/utils.ts diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index 646c3e572..dd38c04af 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -245,6 +245,7 @@ async function handleCreateCLI(options: CreateOptions): Promise { } const knownAttrs = { + agent_environment: 'runtime' as const, agent_language: standardize(AgentLanguage, options.language), agent_framework: standardize(AgentFramework, options.framework), model_provider: standardize(ModelProviderEnum, options.modelProvider), diff --git a/src/cli/commands/deploy/__tests__/utils.test.ts b/src/cli/commands/deploy/__tests__/utils.test.ts index 2ba22c15a..256eed986 100644 --- a/src/cli/commands/deploy/__tests__/utils.test.ts +++ b/src/cli/commands/deploy/__tests__/utils.test.ts @@ -16,6 +16,7 @@ describe('computeDeployAttrs', () => { expect(computeDeployAttrs(projectSpec, 'diff')).toEqual({ runtime_count: 2, + harness_count: 0, memory_count: 1, credential_count: 3, evaluator_count: 1, @@ -31,6 +32,7 @@ describe('computeDeployAttrs', () => { it('returns zeros for empty spec', () => { expect(computeDeployAttrs({}, 'deploy')).toEqual({ runtime_count: 0, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, diff --git a/src/cli/commands/deploy/utils.ts b/src/cli/commands/deploy/utils.ts index c866ded9e..d6362e114 100644 --- a/src/cli/commands/deploy/utils.ts +++ b/src/cli/commands/deploy/utils.ts @@ -3,6 +3,7 @@ import type { DeployMode } from '../../telemetry/schemas/common-shapes'; export const DEFAULT_DEPLOY_ATTRS = { runtime_count: 0, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, @@ -19,6 +20,7 @@ export function computeDeployAttrs(projectSpec: Partial, m const policyEngines = projectSpec.policyEngines ?? []; return { runtime_count: (projectSpec.runtimes ?? []).length, + harness_count: (projectSpec.harnesses ?? []).length, memory_count: (projectSpec.memories ?? []).length, credential_count: (projectSpec.credentials ?? []).length, evaluator_count: (projectSpec.evaluators ?? []).length, diff --git a/src/cli/commands/dev/command.tsx b/src/cli/commands/dev/command.tsx index b935692f3..d60624940 100644 --- a/src/cli/commands/dev/command.tsx +++ b/src/cli/commands/dev/command.tsx @@ -201,6 +201,7 @@ export const registerDev = (program: Command) => { const execResult = await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime' as const, dev_action: 'exec' as const, ui_mode: 'terminal' as const, has_stream: false, @@ -239,6 +240,7 @@ export const registerDev = (program: Command) => { const invokeResult = await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime' as const, dev_action: 'invoke' as const, ui_mode: 'terminal' as const, has_stream: opts.stream ?? false, @@ -301,6 +303,7 @@ export const registerDev = (program: Command) => { const serverResult = await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime' as const, dev_action: 'server' as const, ui_mode: 'terminal' as const, has_stream: false, @@ -353,6 +356,7 @@ export const registerDev = (program: Command) => { if (opts.logs) { // Preview: harness-only projects need deploy then print invoke instructions if (isPreviewEnabled() && supportedAgents.length === 0 && hasHarnesses) { + recorder.set({ agent_environment: 'harness' as const }); if (!opts.skipDeploy) { await runCliDeploy(); } diff --git a/src/cli/commands/invoke/command.tsx b/src/cli/commands/invoke/command.tsx index 0a847eab2..76208aa3d 100644 --- a/src/cli/commands/invoke/command.tsx +++ b/src/cli/commands/invoke/command.tsx @@ -2,7 +2,6 @@ import { ValidationError, serializeResult } from '../../../lib'; import { getErrorMessage } from '../../errors'; import { isPreviewEnabled } from '../../feature-flags'; import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; -import { AgentProtocol, AuthType, standardize } from '../../telemetry/schemas/common-shapes.js'; import { renderTUI } from '../../tui'; import { COMMAND_DESCRIPTIONS } from '../../tui/copy'; import { requireProject, requireTTY } from '../../tui/guards'; @@ -10,6 +9,7 @@ import { parseHeaderFlags } from '../shared/header-utils'; import { type InvokeContext, handleHarnessInvokeByArn, handleInvoke, loadInvokeConfig } from './action'; import { resolvePrompt } from './resolve-prompt'; import type { InvokeOptions, InvokeResult } from './types'; +import { computeInvokeAttrs } from './utils'; import { validateInvokeOptions } from './validate'; import type { Command } from '@commander-js/extra-typings'; import { Text, render } from 'ink'; @@ -30,12 +30,6 @@ function stopSpinner(spinner: NodeJS.Timeout): void { process.stderr.write('\r\x1b[K'); // Clear line } -function resolveProtocol(options: InvokeOptions, projectProtocol?: string): string { - if (projectProtocol) return projectProtocol.toLowerCase(); - if (options.tool) return 'mcp'; - return 'http'; -} - async function handleInvokeCLI(options: InvokeOptions, preloadedContext?: InvokeContext): Promise { const validation = validateInvokeOptions(options); if (!validation.valid) { @@ -261,15 +255,17 @@ export const registerInvoke = (program: Command) => { ) { const result = await withCommandRunTelemetry( 'invoke', - { - has_stream: cliOptions.stream ?? false, - has_session_id: !!cliOptions.sessionId, - auth_type: standardize(AuthType, cliOptions.bearerToken ? 'bearer_token' : 'sigv4'), - agent_protocol: standardize( - AgentProtocol, - resolveProtocol({ tool: cliOptions.tool } as InvokeOptions, agentProtocol) - ), - }, + computeInvokeAttrs({ + preview: isPreviewEnabled(), + harnessName: cliOptions.harness, + harnessArn: cliOptions.harnessArn, + harnessCount: invokeContext?.project.harnesses?.length ?? 0, + runtimeCount: invokeContext?.project.runtimes.length ?? 0, + stream: cliOptions.stream ?? false, + hasSessionId: !!cliOptions.sessionId, + bearerToken: cliOptions.bearerToken, + agentProtocol: agentProtocol ?? (cliOptions.tool ? 'mcp' : undefined), + }), async (): Promise => { if (!resolved.success) { return { success: false, error: new ValidationError(resolved.error ?? 'Prompt resolution failed') }; diff --git a/src/cli/commands/invoke/utils.ts b/src/cli/commands/invoke/utils.ts new file mode 100644 index 000000000..beda64134 --- /dev/null +++ b/src/cli/commands/invoke/utils.ts @@ -0,0 +1,33 @@ +import { AgentEnvironment, AgentProtocol, AuthType, standardize } from '../../telemetry/schemas/common-shapes.js'; + +function isHarnessInvoke(options: { + harnessName?: string; + harnessArn?: string; + harnessCount: number; + runtimeCount: number; +}): boolean { + if (options.harnessName || options.harnessArn) return true; + if (options.harnessCount > 0 && options.runtimeCount === 0) return true; + return false; +} + +export function computeInvokeAttrs(options: { + preview: boolean; + harnessName?: string; + harnessArn?: string; + harnessCount: number; + runtimeCount: number; + stream: boolean; + hasSessionId: boolean; + bearerToken?: string; + agentProtocol?: string; +}) { + const isHarness = options.preview && isHarnessInvoke(options); + return { + agent_environment: standardize(AgentEnvironment, isHarness ? 'harness' : 'runtime'), + has_stream: options.stream, + has_session_id: options.hasSessionId, + auth_type: standardize(AuthType, options.bearerToken ? 'bearer_token' : 'sigv4'), + agent_protocol: isHarness ? undefined : standardize(AgentProtocol, options.agentProtocol ?? 'http'), + }; +} diff --git a/src/cli/telemetry/__tests__/client.test.ts b/src/cli/telemetry/__tests__/client.test.ts index 24263e6ad..bf43d5264 100644 --- a/src/cli/telemetry/__tests__/client.test.ts +++ b/src/cli/telemetry/__tests__/client.test.ts @@ -90,6 +90,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'create', { + agent_environment: 'runtime', agent_language: 'rust' as never, agent_framework: 'strands', model_provider: 'bedrock', @@ -112,6 +113,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'create', { + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', @@ -172,6 +174,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -204,6 +207,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -229,6 +233,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -249,6 +254,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, @@ -277,6 +283,7 @@ describe('withCommandRunTelemetry', () => { await withCommandRunTelemetry( 'dev', { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'terminal', has_stream: false, diff --git a/src/cli/telemetry/schemas/__tests__/command-run.test.ts b/src/cli/telemetry/schemas/__tests__/command-run.test.ts index 561899d7e..a2b58f25e 100644 --- a/src/cli/telemetry/schemas/__tests__/command-run.test.ts +++ b/src/cli/telemetry/schemas/__tests__/command-run.test.ts @@ -39,6 +39,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid deploy attrs', () => { const attrs = { runtime_count: 2, + harness_count: 1, memory_count: 1, credential_count: 0, evaluator_count: 0, @@ -56,6 +57,7 @@ describe('COMMAND_SCHEMAS', () => { expect(() => COMMAND_SCHEMAS.deploy.parse({ runtime_count: -1, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, @@ -73,6 +75,7 @@ describe('COMMAND_SCHEMAS', () => { expect(() => COMMAND_SCHEMAS.deploy.parse({ runtime_count: 1.5, + harness_count: 0, memory_count: 0, credential_count: 0, evaluator_count: 0, @@ -88,6 +91,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid create attrs', () => { const attrs = { + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', @@ -104,6 +108,7 @@ describe('COMMAND_SCHEMAS', () => { it('rejects create attrs with invalid enum value', () => { expect(() => COMMAND_SCHEMAS.create.parse({ + agent_environment: 'runtime', agent_language: 'rust', agent_framework: 'strands', model_provider: 'bedrock', @@ -132,6 +137,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid dev invoke attrs', () => { const attrs = { + agent_environment: 'runtime', dev_action: 'invoke', ui_mode: 'terminal', has_stream: true, @@ -143,6 +149,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts valid dev server browser attrs', () => { const attrs = { + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'browser', has_stream: false, @@ -154,6 +161,7 @@ describe('COMMAND_SCHEMAS', () => { it('accepts dev exec attrs', () => { const attrs = { + agent_environment: 'runtime', dev_action: 'exec', ui_mode: 'terminal', has_stream: false, @@ -166,6 +174,7 @@ describe('COMMAND_SCHEMAS', () => { it('rejects dev attrs with invalid action', () => { expect(() => COMMAND_SCHEMAS.dev.parse({ + agent_environment: 'runtime', dev_action: 'unknown', ui_mode: 'terminal', has_stream: false, @@ -178,6 +187,7 @@ describe('COMMAND_SCHEMAS', () => { it('rejects dev attrs with invalid ui_mode', () => { expect(() => COMMAND_SCHEMAS.dev.parse({ + agent_environment: 'runtime', dev_action: 'server', ui_mode: 'headless', has_stream: false, @@ -216,11 +226,12 @@ describe('type safety', () => { it('no command schema contains arbitrary string fields', () => { for (const [cmd, schema] of Object.entries(COMMAND_SCHEMAS)) { for (const [field, zodType] of Object.entries(schema.shape)) { + const inner = zodType instanceof z.ZodOptional ? zodType.unwrap() : zodType; const safe = - zodType instanceof z.ZodEnum || - zodType instanceof z.ZodBoolean || - zodType instanceof z.ZodNumber || - zodType instanceof z.ZodLiteral; + inner instanceof z.ZodEnum || + inner instanceof z.ZodBoolean || + inner instanceof z.ZodNumber || + inner instanceof z.ZodLiteral; expect(safe, `${cmd}.${field} is an unsafe type`).toBe(true); } } @@ -240,6 +251,7 @@ describe('type safety', () => { describe('resilientParse', () => { it('passes valid attrs through unchanged', () => { const attrs = { + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', @@ -273,14 +285,14 @@ describe('resilientParse', () => { it('defaults missing required fields to unknown', () => { const result = resilientParse(COMMAND_SCHEMAS.create, { agent_language: 'python' }); expect(result.agent_language).toBe('python'); - expect(result.agent_framework).toBe('unknown'); - expect(result.model_provider).toBe('unknown'); + expect(result.agent_environment).toBe('unknown'); + expect(result.has_agent).toBe('unknown'); }); it('defaults all fields to unknown when all are invalid', () => { const result = resilientParse(COMMAND_SCHEMAS.create, {}); for (const value of Object.values(result)) { - expect(value).toBe('unknown'); + expect(value === 'unknown' || value === undefined).toBe(true); } }); diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index 631fccb11..8b0cbe477 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -1,4 +1,5 @@ import { + AgentEnvironment, AgentFramework, AgentLanguage, AgentProtocol, @@ -33,14 +34,15 @@ import { import { z } from 'zod'; const CreateAttrs = safeSchema({ - agent_language: AgentLanguage, - agent_framework: AgentFramework, - model_provider: ModelProvider, - memory_type: MemoryType, - agent_protocol: AgentProtocol, - build_type: BuildType, - agent_type: AgentType, - network_mode: NetworkMode, + agent_environment: AgentEnvironment, + agent_language: AgentLanguage.optional(), + agent_framework: AgentFramework.optional(), + model_provider: ModelProvider.optional(), + memory_type: MemoryType.optional(), + agent_protocol: AgentProtocol.optional(), + build_type: BuildType.optional(), + agent_type: AgentType.optional(), + network_mode: NetworkMode.optional(), has_agent: z.boolean(), }); @@ -95,6 +97,7 @@ const AddPolicyAttrs = safeSchema({ const DeployAttrs = safeSchema({ runtime_count: Count, + harness_count: Count, memory_count: Count, credential_count: Count, evaluator_count: Count, @@ -107,18 +110,20 @@ const DeployAttrs = safeSchema({ }); const DevAttrs = safeSchema({ + agent_environment: AgentEnvironment, dev_action: DevAction, ui_mode: UiMode, has_stream: z.boolean(), - agent_protocol: AgentProtocol, + agent_protocol: AgentProtocol.optional(), invoke_count: Count, }); const InvokeAttrs = safeSchema({ + agent_environment: AgentEnvironment, has_stream: z.boolean(), has_session_id: z.boolean(), auth_type: AuthType, - agent_protocol: AgentProtocol, + agent_protocol: AgentProtocol.optional(), }); const StatusAttrs = safeSchema({ filter_type: FilterType, filter_state: FilterState }); diff --git a/src/cli/telemetry/schemas/common-shapes.ts b/src/cli/telemetry/schemas/common-shapes.ts index 552abfd5d..0d1d0085c 100644 --- a/src/cli/telemetry/schemas/common-shapes.ts +++ b/src/cli/telemetry/schemas/common-shapes.ts @@ -3,7 +3,8 @@ import { z } from 'zod'; // Type-safe schema builder: rejects z.string() at compile time. // Only z.enum(), z.boolean(), z.number(), and z.literal() are allowed as field types. // eslint-disable-next-line @typescript-eslint/no-explicit-any -type SafeField = z.ZodEnum | z.ZodBoolean | z.ZodNumber | z.ZodLiteral; +type BaseSafeField = z.ZodEnum | z.ZodBoolean | z.ZodNumber | z.ZodLiteral; +type SafeField = BaseSafeField | z.ZodOptional; export function safeSchema>(shape: T) { return z.object(shape); } @@ -71,6 +72,7 @@ export const FilterType = z.enum([ 'harness', 'none', ]); +export const AgentEnvironment = z.enum(['harness', 'runtime']); export const AgentFramework = z.enum(['strands', 'langchain_langgraph', 'googleadk', 'openaiagents']); export const GatewayTargetHost = z.enum(['lambda', 'agentcoreruntime']); export const GatewayTargetType = z.enum([ @@ -152,6 +154,7 @@ export type DeployMode = z.infer; Keys are the field names as they appear in emitted metrics. */ export const ATTRIBUTES = { + agent_environment: AgentEnvironment, dev_action: DevAction, agent_type: AgentType, attach_gateway_count: Count, diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts index 3c5386476..1d560f31b 100644 --- a/src/cli/tui/screens/create/useCreateFlow.ts +++ b/src/cli/tui/screens/create/useCreateFlow.ts @@ -26,6 +26,7 @@ import { credentialPrimitive } from '../../../primitives/registry'; import { createDefaultProjectSpec } from '../../../project'; import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { + AgentEnvironment, AgentFramework, AgentLanguage, AgentProtocol, @@ -260,6 +261,7 @@ export function useCreateFlow(cwd: string): CreateFlowState { if (phase !== 'running') return; const attrs = { + agent_environment: standardize(AgentEnvironment, addHarnessConfig ? 'harness' : 'runtime'), agent_language: standardize(AgentLanguage, addAgentConfig?.language ?? 'Python'), agent_framework: standardize(AgentFramework, addAgentConfig?.framework), model_provider: standardize(ModelProvider, addAgentConfig?.modelProvider), diff --git a/src/cli/tui/screens/invoke/useInvokeFlow.ts b/src/cli/tui/screens/invoke/useInvokeFlow.ts index a770cc7ae..0e80c0b3c 100644 --- a/src/cli/tui/screens/invoke/useInvokeFlow.ts +++ b/src/cli/tui/screens/invoke/useInvokeFlow.ts @@ -22,6 +22,7 @@ import { mcpListTools, } from '../../../aws'; import { invokeHarness } from '../../../aws/agentcore-harness'; +import { computeInvokeAttrs } from '../../../commands/invoke/utils'; import { ANSI } from '../../../constants'; import { getErrorMessage } from '../../../errors'; import { isPreviewEnabled } from '../../../feature-flags'; @@ -35,7 +36,6 @@ import { } from '../../../operations/fetch-access'; import { generateSessionId } from '../../../operations/session'; import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; -import { AgentProtocol, AuthType, standardize } from '../../../telemetry/schemas/common-shapes.js'; import { useCallback, useEffect, useRef, useState } from 'react'; /** Structured message part for rich AGUI event rendering */ @@ -138,12 +138,16 @@ export function useInvokeFlow(options: InvokeFlowOptions = {}): InvokeFlowState const result = await withCommandRunTelemetry( 'invoke', - { - has_stream: true, - has_session_id: !!initialSessionId, - auth_type: standardize(AuthType, initialBearerToken ? 'bearer_token' : 'sigv4'), - agent_protocol: standardize(AgentProtocol, firstProtocol), - }, + computeInvokeAttrs({ + preview: isPreviewEnabled(), + harnessName: initialHarnessName, + harnessCount: project?.harnesses?.length ?? 0, + runtimeCount: project?.runtimes.length ?? 0, + stream: true, + hasSessionId: !!initialSessionId, + bearerToken: initialBearerToken, + agentProtocol: firstProtocol, + }), async () => { if (!project) { return { success: false as const, error: new ResourceNotFoundError('No agentcore project found.') }; From 6f35946770c180b320cca4dc6f3cc4306ddfc7f6 Mon Sep 17 00:00:00 2001 From: hkobew Date: Wed, 27 May 2026 23:02:10 -0400 Subject: [PATCH 02/10] test(telemetry): assert agent_environment in existing integ tests --- integ-tests/create-edge-cases.test.ts | 2 ++ integ-tests/create-frameworks.test.ts | 1 + integ-tests/dev-server.test.ts | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/integ-tests/create-edge-cases.test.ts b/integ-tests/create-edge-cases.test.ts index 93849328c..a05a51938 100644 --- a/integ-tests/create-edge-cases.test.ts +++ b/integ-tests/create-edge-cases.test.ts @@ -39,6 +39,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', exit_reason: 'failure', error_name: 'ValidationError', error_source: 'user', + agent_environment: 'runtime', agent_language: 'python', has_agent: 'true', }); @@ -143,6 +144,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', telemetry.assertMetricEmitted({ command: 'create', exit_reason: 'success', + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'strands', model_provider: 'bedrock', diff --git a/integ-tests/create-frameworks.test.ts b/integ-tests/create-frameworks.test.ts index 2fcce842b..410d137b2 100644 --- a/integ-tests/create-frameworks.test.ts +++ b/integ-tests/create-frameworks.test.ts @@ -69,6 +69,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen telemetry.assertMetricEmitted({ command: 'create', exit_reason: 'success', + agent_environment: 'runtime', agent_language: 'python', agent_framework: 'langchain_langgraph', model_provider: 'bedrock', diff --git a/integ-tests/dev-server.test.ts b/integ-tests/dev-server.test.ts index f2babe662..940baa0d2 100644 --- a/integ-tests/dev-server.test.ts +++ b/integ-tests/dev-server.test.ts @@ -109,6 +109,7 @@ describe('integration: dev server', () => { command: 'dev', dev_action: 'server', ui_mode: 'terminal', + agent_environment: 'runtime', exit_reason: 'success', }); telemetry.clearEntries(); @@ -123,6 +124,7 @@ describe('integration: dev server', () => { command: 'dev', dev_action: 'invoke', ui_mode: 'terminal', + agent_environment: 'runtime', exit_reason: 'success', agent_protocol: 'http', }); @@ -135,6 +137,7 @@ describe('integration: dev server', () => { telemetry.assertMetricEmitted({ command: 'dev', dev_action: 'invoke', + agent_environment: 'runtime', exit_reason: 'failure', }); @@ -162,6 +165,7 @@ describe('integration: dev server', () => { telemetry.assertMetricEmitted({ command: 'dev', dev_action: 'server', + agent_environment: 'runtime', exit_reason: 'failure', }); }, From b67947a13e1df9d85b5d18e3ee85a0618123a987 Mon Sep 17 00:00:00 2001 From: hkobew Date: Wed, 27 May 2026 23:20:52 -0400 Subject: [PATCH 03/10] feat(telemetry): add telemetry to harness create and integ tests for harness flows --- integ-tests/create-edge-cases.test.ts | 35 +++++++ integ-tests/dev-server.test.ts | 40 ++++++++ src/cli/commands/create/command.tsx | 131 +++++++++++++------------- 3 files changed, 141 insertions(+), 65 deletions(-) diff --git a/integ-tests/create-edge-cases.test.ts b/integ-tests/create-edge-cases.test.ts index a05a51938..9c7d7e3dd 100644 --- a/integ-tests/create-edge-cases.test.ts +++ b/integ-tests/create-edge-cases.test.ts @@ -196,3 +196,38 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', }); }); }); + +const isPreviewBuild = process.env.BUILD_PREVIEW === '1'; + +describe.skipIf(!isPreviewBuild || !prereqs.npm || !prereqs.git)('integration: create harness project', () => { + let testDir: string; + const telemetry = createTelemetryHelper(); + + beforeAll(async () => { + testDir = join(tmpdir(), `agentcore-integ-create-harness-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + }); + + afterAll(async () => { + telemetry.destroy(); + await rm(testDir, { recursive: true, force: true }); + }); + + it('creates a harness project with defaults', async () => { + const name = `Hrn${Date.now().toString().slice(-6)}`; + const result = await runCLI(['create', '--name', name, '--model-provider', 'bedrock', '--json'], testDir, { + env: telemetry.env, + }); + + expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + telemetry.assertMetricEmitted({ + command: 'create', + exit_reason: 'success', + agent_environment: 'harness', + has_agent: 'false', + }); + }); +}); diff --git a/integ-tests/dev-server.test.ts b/integ-tests/dev-server.test.ts index 940baa0d2..0a39da1bf 100644 --- a/integ-tests/dev-server.test.ts +++ b/integ-tests/dev-server.test.ts @@ -172,3 +172,43 @@ describe('integration: dev server', () => { 15000 ); }); + +const isPreviewBuild = process.env.BUILD_PREVIEW === '1'; + +describe.skipIf(!isPreviewBuild || !hasNpm || !hasGit || !hasUv)('integration: dev with harness-only project', () => { + const telemetry = createTelemetryHelper(); + let projectPath: string; + + beforeAll(async () => { + const dir = join(tmpdir(), `agentcore-dev-harness-${Date.now()}`); + await mkdir(dir, { recursive: true }); + + // Create a harness-only project + const createResult = await runCLI( + ['create', '--name', 'DevHarness', '--model-provider', 'bedrock', '--json'], + dir, + { env: telemetry.env } + ); + const json = JSON.parse(createResult.stdout); + projectPath = json.projectPath; + }); + + afterAll(async () => { + telemetry.destroy(); + if (projectPath) await rm(projectPath, { recursive: true, force: true }); + }); + + it('dev --logs on harness-only project succeeds without a local server', async () => { + telemetry.clearEntries(); + const result = await runCLI(['dev', '--logs', '--skip-deploy'], projectPath, { env: telemetry.env }); + + expect(result.exitCode).toBe(0); + + telemetry.assertMetricEmitted({ + command: 'dev', + dev_action: 'server', + agent_environment: 'harness', + exit_reason: 'success', + }); + }); +}); diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index dd38c04af..b766f9a4f 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -12,7 +12,7 @@ import { LIFECYCLE_TIMEOUT_MAX, LIFECYCLE_TIMEOUT_MIN } from '../../../schema'; import { getErrorMessage } from '../../errors'; import { isPreviewEnabled } from '../../feature-flags'; import { harnessPrimitive } from '../../primitives/registry'; -import { runCliCommand } from '../../telemetry/cli-command-run.js'; +import { runCliCommand, withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { AgentFramework, AgentLanguage, @@ -139,78 +139,79 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { const name = options.name ?? options.projectName; const projectName = options.projectName ?? name; - const validation = validateCreateHarnessOptions( - { - name, - projectName, - modelProvider: options.modelProvider, - modelId: options.modelId, - apiKeyArn: options.apiKeyArn, - }, - cwd - ); - if (!validation.valid) { - if (options.json) { - console.log(JSON.stringify({ success: false, error: validation.error })); - } else { - console.error(validation.error); - } - process.exit(1); - } + const result = await withCommandRunTelemetry( + 'create', + { agent_environment: 'harness' as const, has_agent: false }, + async () => { + const validation = validateCreateHarnessOptions( + { + name, + projectName, + modelProvider: options.modelProvider, + modelId: options.modelId, + apiKeyArn: options.apiKeyArn, + }, + cwd + ); + if (!validation.valid) { + return { success: false as const, error: new ValidationError(validation.error!) }; + } - // Progress callback - const green = '\x1b[32m'; - const reset = '\x1b[0m'; - const onProgress: ProgressCallback | undefined = options.json - ? undefined - : (step, status) => { - if (status === 'done') console.log(`${green}[done]${reset} ${step}`); - else if (status === 'error') console.log(`\x1b[31m[error]${reset} ${step}`); - }; + // Progress callback + const green = '\x1b[32m'; + const reset = '\x1b[0m'; + const onProgress: ProgressCallback | undefined = options.json + ? undefined + : (step, status) => { + if (status === 'done') console.log(`${green}[done]${reset} ${step}`); + else if (status === 'error') console.log(`\x1b[31m[error]${reset} ${step}`); + }; - const provider = ( - options.modelProvider ? normalizeHarnessModelProvider(options.modelProvider) : 'bedrock' - ) as HarnessModelProvider; - const defaultModelIds: Record = { - bedrock: 'global.anthropic.claude-sonnet-4-6', - open_ai: 'gpt-5', - gemini: 'gemini-2.5-flash', - }; - const modelId = options.modelId ?? defaultModelIds[provider] ?? 'global.anthropic.claude-sonnet-4-6'; - - const containerOption = harnessPrimitive!.parseContainerFlag(options.container); - - const result = await createProjectWithHarness({ - name: name!, - projectName: projectName!, - cwd, - modelProvider: provider, - modelId, - apiKeyArn: options.apiKeyArn, - containerUri: containerOption.containerUri, - dockerfilePath: containerOption.dockerfilePath, - skipMemory: options.harnessMemory === false, - maxIterations: options.maxIterations ? Number(options.maxIterations) : undefined, - maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined, - timeoutSeconds: options.timeout ? Number(options.timeout) : undefined, - truncationStrategy: options.truncationStrategy as 'sliding_window' | 'summarization' | undefined, - networkMode: options.networkMode as NetworkMode | undefined, - subnets: parseCommaSeparatedList(options.subnets), - securityGroups: parseCommaSeparatedList(options.securityGroups), - idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined, - maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined, - sessionStoragePath: options.sessionStorageMountPath, - skipGit: options.skipGit, - skipInstall: options.skipInstall, - onProgress, - }); + const provider = ( + options.modelProvider ? normalizeHarnessModelProvider(options.modelProvider) : 'bedrock' + ) as HarnessModelProvider; + const defaultModelIds: Record = { + bedrock: 'global.anthropic.claude-sonnet-4-6', + open_ai: 'gpt-5', + gemini: 'gemini-2.5-flash', + }; + const modelId = options.modelId ?? defaultModelIds[provider] ?? 'global.anthropic.claude-sonnet-4-6'; + + const containerOption = harnessPrimitive!.parseContainerFlag(options.container); + + return createProjectWithHarness({ + name: name!, + projectName: projectName!, + cwd, + modelProvider: provider, + modelId, + apiKeyArn: options.apiKeyArn, + containerUri: containerOption.containerUri, + dockerfilePath: containerOption.dockerfilePath, + skipMemory: options.harnessMemory === false, + maxIterations: options.maxIterations ? Number(options.maxIterations) : undefined, + maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined, + timeoutSeconds: options.timeout ? Number(options.timeout) : undefined, + truncationStrategy: options.truncationStrategy as 'sliding_window' | 'summarization' | undefined, + networkMode: options.networkMode as NetworkMode | undefined, + subnets: parseCommaSeparatedList(options.subnets), + securityGroups: parseCommaSeparatedList(options.securityGroups), + idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined, + maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined, + sessionStoragePath: options.sessionStorageMountPath, + skipGit: options.skipGit, + skipInstall: options.skipInstall, + onProgress, + }); + } + ); if (options.json) { console.log(JSON.stringify(result)); } else if (result.success) { printCreateHarnessSummary(projectName!, name!); } else { - console.error(result.error); + console.error((result as { error?: Error }).error?.message ?? 'Create failed'); } process.exit(result.success ? 0 : 1); } From 9e70a34c559fccd176b6da85c48838a19dac0423 Mon Sep 17 00:00:00 2001 From: hkobew Date: Wed, 27 May 2026 23:46:45 -0400 Subject: [PATCH 04/10] fix(telemetry): fix harness create serialization and add computeInvokeAttrs tests --- integ-tests/create-edge-cases.test.ts | 2 +- src/cli/commands/create/command.tsx | 6 +- .../commands/invoke/__tests__/utils.test.ts | 100 ++++++++++++++++++ src/cli/tui/screens/create/useCreateFlow.ts | 25 +++-- src/cli/tui/screens/invoke/useInvokeFlow.ts | 2 +- 5 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 src/cli/commands/invoke/__tests__/utils.test.ts diff --git a/integ-tests/create-edge-cases.test.ts b/integ-tests/create-edge-cases.test.ts index 9c7d7e3dd..d7982eed8 100644 --- a/integ-tests/create-edge-cases.test.ts +++ b/integ-tests/create-edge-cases.test.ts @@ -227,7 +227,7 @@ describe.skipIf(!isPreviewBuild || !prereqs.npm || !prereqs.git)('integration: c command: 'create', exit_reason: 'success', agent_environment: 'harness', - has_agent: 'false', + has_agent: 'true', }); }); }); diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index b766f9a4f..c328f295f 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -141,7 +141,7 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { const result = await withCommandRunTelemetry( 'create', - { agent_environment: 'harness' as const, has_agent: false }, + { agent_environment: 'harness' as const, has_agent: true }, async () => { const validation = validateCreateHarnessOptions( { @@ -207,11 +207,11 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { ); if (options.json) { - console.log(JSON.stringify(result)); + console.log(JSON.stringify(serializeResult(result))); } else if (result.success) { printCreateHarnessSummary(projectName!, name!); } else { - console.error((result as { error?: Error }).error?.message ?? 'Create failed'); + console.error(result.error instanceof Error ? result.error.message : 'Create failed'); } process.exit(result.success ? 0 : 1); } diff --git a/src/cli/commands/invoke/__tests__/utils.test.ts b/src/cli/commands/invoke/__tests__/utils.test.ts new file mode 100644 index 000000000..7187c7409 --- /dev/null +++ b/src/cli/commands/invoke/__tests__/utils.test.ts @@ -0,0 +1,100 @@ +import { computeInvokeAttrs } from '../utils'; +import { describe, expect, it } from 'vitest'; + +describe('computeInvokeAttrs', () => { + it('returns runtime when preview is false regardless of harness flags', () => { + const attrs = computeInvokeAttrs({ + preview: false, + harnessName: 'my-harness', + harnessCount: 1, + runtimeCount: 0, + stream: true, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('runtime'); + expect(attrs.agent_protocol).toBe('http'); + }); + + it('returns harness when harnessName is set and preview is true', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessName: 'my-harness', + harnessCount: 1, + runtimeCount: 1, + stream: false, + hasSessionId: true, + }); + expect(attrs.agent_environment).toBe('harness'); + expect(attrs.agent_protocol).toBeUndefined(); + expect(attrs.has_session_id).toBe(true); + }); + + it('returns harness when harnessArn is set and preview is true', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessArn: 'arn:aws:bedrock:us-east-1:123:harness/h1', + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('harness'); + expect(attrs.agent_protocol).toBeUndefined(); + }); + + it('returns harness when project has only harnesses', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessCount: 2, + runtimeCount: 0, + stream: false, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('harness'); + }); + + it('returns runtime for mixed project without explicit harness flag', () => { + const attrs = computeInvokeAttrs({ + preview: true, + harnessCount: 1, + runtimeCount: 1, + stream: false, + hasSessionId: false, + }); + expect(attrs.agent_environment).toBe('runtime'); + expect(attrs.agent_protocol).toBe('http'); + }); + + it('passes auth_type based on bearerToken', () => { + const withToken = computeInvokeAttrs({ + preview: false, + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + bearerToken: 'tok', + }); + expect(withToken.auth_type).toBe('bearer_token'); + + const withoutToken = computeInvokeAttrs({ + preview: false, + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + }); + expect(withoutToken.auth_type).toBe('sigv4'); + }); + + it('uses provided agentProtocol for runtime', () => { + const attrs = computeInvokeAttrs({ + preview: false, + harnessCount: 0, + runtimeCount: 1, + stream: false, + hasSessionId: false, + agentProtocol: 'MCP', + }); + expect(attrs.agent_protocol).toBe('mcp'); + }); +}); diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts index 1d560f31b..331462ad1 100644 --- a/src/cli/tui/screens/create/useCreateFlow.ts +++ b/src/cli/tui/screens/create/useCreateFlow.ts @@ -260,17 +260,22 @@ export function useCreateFlow(cwd: string): CreateFlowState { useEffect(() => { if (phase !== 'running') return; + const isHarness = addHarnessConfig !== null; const attrs = { - agent_environment: standardize(AgentEnvironment, addHarnessConfig ? 'harness' : 'runtime'), - agent_language: standardize(AgentLanguage, addAgentConfig?.language ?? 'Python'), - agent_framework: standardize(AgentFramework, addAgentConfig?.framework), - model_provider: standardize(ModelProvider, addAgentConfig?.modelProvider), - memory_type: standardize(MemoryEnum, addAgentConfig?.memory ?? 'none'), - agent_protocol: standardize(AgentProtocol, addAgentConfig?.protocol ?? 'HTTP'), - build_type: standardize(BuildType, addAgentConfig?.buildType ?? 'CodeZip'), - agent_type: standardize(AgentType, addAgentConfig?.agentType ?? 'create'), - network_mode: standardize(NetworkMode, addAgentConfig?.networkMode ?? 'PUBLIC'), - has_agent: addAgentConfig !== null, + agent_environment: standardize(AgentEnvironment, isHarness ? 'harness' : 'runtime'), + has_agent: true, + ...(isHarness + ? {} + : { + agent_language: standardize(AgentLanguage, addAgentConfig?.language ?? 'Python'), + agent_framework: standardize(AgentFramework, addAgentConfig?.framework), + model_provider: standardize(ModelProvider, addAgentConfig?.modelProvider), + memory_type: standardize(MemoryEnum, addAgentConfig?.memory ?? 'none'), + agent_protocol: standardize(AgentProtocol, addAgentConfig?.protocol ?? 'HTTP'), + build_type: standardize(BuildType, addAgentConfig?.buildType ?? 'CodeZip'), + agent_type: standardize(AgentType, addAgentConfig?.agentType ?? 'create'), + network_mode: standardize(NetworkMode, addAgentConfig?.networkMode ?? 'PUBLIC'), + }), }; const run = async (): Promise<{ success: true } | { success: false; error: Error }> => { diff --git a/src/cli/tui/screens/invoke/useInvokeFlow.ts b/src/cli/tui/screens/invoke/useInvokeFlow.ts index 0e80c0b3c..44481a39d 100644 --- a/src/cli/tui/screens/invoke/useInvokeFlow.ts +++ b/src/cli/tui/screens/invoke/useInvokeFlow.ts @@ -142,7 +142,7 @@ export function useInvokeFlow(options: InvokeFlowOptions = {}): InvokeFlowState preview: isPreviewEnabled(), harnessName: initialHarnessName, harnessCount: project?.harnesses?.length ?? 0, - runtimeCount: project?.runtimes.length ?? 0, + runtimeCount: project?.runtimes?.length ?? 0, stream: true, hasSessionId: !!initialSessionId, bearerToken: initialBearerToken, From 9d84af31eb102aff53e85f87d1833d7205b343f0 Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 28 May 2026 08:54:17 -0400 Subject: [PATCH 05/10] fix(test): add link to issue for test asserting broken behavior --- integ-tests/dev-server.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/integ-tests/dev-server.test.ts b/integ-tests/dev-server.test.ts index 0a39da1bf..0a7fea149 100644 --- a/integ-tests/dev-server.test.ts +++ b/integ-tests/dev-server.test.ts @@ -198,17 +198,18 @@ describe.skipIf(!isPreviewBuild || !hasNpm || !hasGit || !hasUv)('integration: d if (projectPath) await rm(projectPath, { recursive: true, force: true }); }); - it('dev --logs on harness-only project succeeds without a local server', async () => { + // This test currently fails due to https://github.com/aws/agentcore-cli/issues/1406 + it.skip('dev --logs on harness-only project should fail with validation error', async () => { telemetry.clearEntries(); const result = await runCLI(['dev', '--logs', '--skip-deploy'], projectPath, { env: telemetry.env }); - expect(result.exitCode).toBe(0); + expect(result.exitCode).toBe(1); telemetry.assertMetricEmitted({ command: 'dev', dev_action: 'server', agent_environment: 'harness', - exit_reason: 'success', + exit_reason: 'failure', }); }); }); From b2f15e1a81b4af3134a7eb90b91f666a829fc022 Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 28 May 2026 09:01:01 -0400 Subject: [PATCH 06/10] fix: use shared ANSI constants instead of inline escape codes --- src/cli/commands/create/command.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index c328f295f..6be33b45e 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -9,6 +9,7 @@ import type { TargetLanguage, } from '../../../schema'; import { LIFECYCLE_TIMEOUT_MAX, LIFECYCLE_TIMEOUT_MIN } from '../../../schema'; +import { ANSI } from '../../constants'; import { getErrorMessage } from '../../errors'; import { isPreviewEnabled } from '../../feature-flags'; import { harnessPrimitive } from '../../primitives/registry'; @@ -158,13 +159,11 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { } // Progress callback - const green = '\x1b[32m'; - const reset = '\x1b[0m'; const onProgress: ProgressCallback | undefined = options.json ? undefined : (step, status) => { - if (status === 'done') console.log(`${green}[done]${reset} ${step}`); - else if (status === 'error') console.log(`\x1b[31m[error]${reset} ${step}`); + if (status === 'done') console.log(`${ANSI.green}[done]${ANSI.reset} ${step}`); + else if (status === 'error') console.log(`${ANSI.red}[error]${ANSI.reset} ${step}`); }; const provider = ( From 80636764656ddf2a690795e7a7c8e81ae512ad6e Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 28 May 2026 09:05:57 -0400 Subject: [PATCH 07/10] fix: make shared fields among environments required --- src/cli/commands/create/command.tsx | 9 +++++++- src/cli/telemetry/schemas/command-run.ts | 8 +++---- src/cli/tui/screens/create/useCreateFlow.ts | 24 +++++++++++++++++---- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index 6be33b45e..f6fa29d1f 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -142,7 +142,14 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { const result = await withCommandRunTelemetry( 'create', - { agent_environment: 'harness' as const, has_agent: true }, + { + agent_environment: 'harness' as const, + has_agent: true, + model_provider: standardize(ModelProviderEnum, options.modelProvider ?? 'bedrock'), + memory_type: standardize(MemoryType, options.harnessMemory === false ? 'none' : 'shortterm'), + build_type: standardize(TelemetryBuildType, options.container ? 'container' : 'codezip'), + network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'), + }, async () => { const validation = validateCreateHarnessOptions( { diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index 8b0cbe477..3db5aca82 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -37,12 +37,12 @@ const CreateAttrs = safeSchema({ agent_environment: AgentEnvironment, agent_language: AgentLanguage.optional(), agent_framework: AgentFramework.optional(), - model_provider: ModelProvider.optional(), - memory_type: MemoryType.optional(), + model_provider: ModelProvider, + memory_type: MemoryType, agent_protocol: AgentProtocol.optional(), - build_type: BuildType.optional(), + build_type: BuildType, agent_type: AgentType.optional(), - network_mode: NetworkMode.optional(), + network_mode: NetworkMode, has_agent: z.boolean(), }); diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts index 331462ad1..9adf65c6f 100644 --- a/src/cli/tui/screens/create/useCreateFlow.ts +++ b/src/cli/tui/screens/create/useCreateFlow.ts @@ -264,17 +264,33 @@ export function useCreateFlow(cwd: string): CreateFlowState { const attrs = { agent_environment: standardize(AgentEnvironment, isHarness ? 'harness' : 'runtime'), has_agent: true, + model_provider: standardize( + ModelProvider, + isHarness ? addHarnessConfig?.modelProvider : addAgentConfig?.modelProvider + ), + memory_type: standardize( + MemoryEnum, + isHarness ? (addHarnessConfig?.skipMemory ? 'none' : 'shortterm') : (addAgentConfig?.memory ?? 'none') + ), + build_type: standardize( + BuildType, + isHarness + ? addHarnessConfig?.containerMode && addHarnessConfig.containerMode !== 'none' + ? 'container' + : 'codezip' + : (addAgentConfig?.buildType ?? 'CodeZip') + ), + network_mode: standardize( + NetworkMode, + isHarness ? (addHarnessConfig?.networkMode ?? 'PUBLIC') : (addAgentConfig?.networkMode ?? 'PUBLIC') + ), ...(isHarness ? {} : { agent_language: standardize(AgentLanguage, addAgentConfig?.language ?? 'Python'), agent_framework: standardize(AgentFramework, addAgentConfig?.framework), - model_provider: standardize(ModelProvider, addAgentConfig?.modelProvider), - memory_type: standardize(MemoryEnum, addAgentConfig?.memory ?? 'none'), agent_protocol: standardize(AgentProtocol, addAgentConfig?.protocol ?? 'HTTP'), - build_type: standardize(BuildType, addAgentConfig?.buildType ?? 'CodeZip'), agent_type: standardize(AgentType, addAgentConfig?.agentType ?? 'create'), - network_mode: standardize(NetworkMode, addAgentConfig?.networkMode ?? 'PUBLIC'), }), }; From d06522a4476f47f5033cd838ab2cf0d65840e505 Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 28 May 2026 09:18:19 -0400 Subject: [PATCH 08/10] fix(telemetry): avoid hardcoding has_agent to true --- src/cli/tui/screens/create/useCreateFlow.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts index 9adf65c6f..411a2145e 100644 --- a/src/cli/tui/screens/create/useCreateFlow.ts +++ b/src/cli/tui/screens/create/useCreateFlow.ts @@ -263,7 +263,8 @@ export function useCreateFlow(cwd: string): CreateFlowState { const isHarness = addHarnessConfig !== null; const attrs = { agent_environment: standardize(AgentEnvironment, isHarness ? 'harness' : 'runtime'), - has_agent: true, + // true when either an agent or harness config is set (non-null/non-undefined) + has_agent: Boolean(addAgentConfig) || Boolean(addHarnessConfig), model_provider: standardize( ModelProvider, isHarness ? addHarnessConfig?.modelProvider : addAgentConfig?.modelProvider From 127a0790e113fd8ee91a9fb79da50d6fa6dc10c4 Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 28 May 2026 09:23:27 -0400 Subject: [PATCH 09/10] fix(telemetry): use short and long term for harness memory --- src/cli/commands/create/command.tsx | 2 +- src/cli/tui/screens/create/useCreateFlow.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index f6fa29d1f..536a5e77d 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -146,7 +146,7 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { agent_environment: 'harness' as const, has_agent: true, model_provider: standardize(ModelProviderEnum, options.modelProvider ?? 'bedrock'), - memory_type: standardize(MemoryType, options.harnessMemory === false ? 'none' : 'shortterm'), + memory_type: standardize(MemoryType, options.harnessMemory === false ? 'none' : 'longandshortterm'), build_type: standardize(TelemetryBuildType, options.container ? 'container' : 'codezip'), network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'), }, diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts index 411a2145e..0e28764c0 100644 --- a/src/cli/tui/screens/create/useCreateFlow.ts +++ b/src/cli/tui/screens/create/useCreateFlow.ts @@ -271,7 +271,7 @@ export function useCreateFlow(cwd: string): CreateFlowState { ), memory_type: standardize( MemoryEnum, - isHarness ? (addHarnessConfig?.skipMemory ? 'none' : 'shortterm') : (addAgentConfig?.memory ?? 'none') + isHarness ? (addHarnessConfig?.skipMemory ? 'none' : 'longandshortterm') : (addAgentConfig?.memory ?? 'none') ), build_type: standardize( BuildType, From 7187e11a6bd2ec7f87a9a0261b66a33d2a4652e4 Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 28 May 2026 09:27:19 -0400 Subject: [PATCH 10/10] fix(telemetry): make build_type optional, not relevant for harness --- src/cli/commands/create/command.tsx | 1 - src/cli/telemetry/schemas/command-run.ts | 2 +- src/cli/tui/screens/create/useCreateFlow.ts | 9 +-------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index 536a5e77d..dbc1e096f 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -147,7 +147,6 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise { has_agent: true, model_provider: standardize(ModelProviderEnum, options.modelProvider ?? 'bedrock'), memory_type: standardize(MemoryType, options.harnessMemory === false ? 'none' : 'longandshortterm'), - build_type: standardize(TelemetryBuildType, options.container ? 'container' : 'codezip'), network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'), }, async () => { diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index 3db5aca82..e5a0cd86e 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -40,7 +40,7 @@ const CreateAttrs = safeSchema({ model_provider: ModelProvider, memory_type: MemoryType, agent_protocol: AgentProtocol.optional(), - build_type: BuildType, + build_type: BuildType.optional(), agent_type: AgentType.optional(), network_mode: NetworkMode, has_agent: z.boolean(), diff --git a/src/cli/tui/screens/create/useCreateFlow.ts b/src/cli/tui/screens/create/useCreateFlow.ts index 0e28764c0..743bceb13 100644 --- a/src/cli/tui/screens/create/useCreateFlow.ts +++ b/src/cli/tui/screens/create/useCreateFlow.ts @@ -273,14 +273,7 @@ export function useCreateFlow(cwd: string): CreateFlowState { MemoryEnum, isHarness ? (addHarnessConfig?.skipMemory ? 'none' : 'longandshortterm') : (addAgentConfig?.memory ?? 'none') ), - build_type: standardize( - BuildType, - isHarness - ? addHarnessConfig?.containerMode && addHarnessConfig.containerMode !== 'none' - ? 'container' - : 'codezip' - : (addAgentConfig?.buildType ?? 'CodeZip') - ), + build_type: isHarness ? undefined : standardize(BuildType, addAgentConfig?.buildType ?? 'CodeZip'), network_mode: standardize( NetworkMode, isHarness ? (addHarnessConfig?.networkMode ?? 'PUBLIC') : (addAgentConfig?.networkMode ?? 'PUBLIC')