feat(telemetry): support agent_environment for runtime/harness distinction

integ-tests/create-edge-cases.test.ts

@@ -39,6 +39,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
         exit_reason: 'failure',
         error_name: 'ValidationError',
         error_source: 'user',
+        agent_environment: 'runtime',
         agent_language: 'python',
         has_agent: 'true',
       });
@@ -143,6 +144,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
       telemetry.assertMetricEmitted({
         command: 'create',
         exit_reason: 'success',
+        agent_environment: 'runtime',
         agent_language: 'python',
         agent_framework: 'strands',
         model_provider: 'bedrock',
@@ -194,3 +196,38 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
     });
   });
 });
+
+const isPreviewBuild = process.env.BUILD_PREVIEW === '1';
+
+describe.skipIf(!isPreviewBuild || !prereqs.npm || !prereqs.git)('integration: create harness project', () => {
+  let testDir: string;
+  const telemetry = createTelemetryHelper();
+
+  beforeAll(async () => {
+    testDir = join(tmpdir(), `agentcore-integ-create-harness-${randomUUID()}`);
+    await mkdir(testDir, { recursive: true });
+  });
+
+  afterAll(async () => {
+    telemetry.destroy();
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  it('creates a harness project with defaults', async () => {
+    const name = `Hrn${Date.now().toString().slice(-6)}`;
+    const result = await runCLI(['create', '--name', name, '--model-provider', 'bedrock', '--json'], testDir, {
+      env: telemetry.env,
+    });
+
+    expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
+    const json = JSON.parse(result.stdout);
+    expect(json.success).toBe(true);
+
+    telemetry.assertMetricEmitted({
+      command: 'create',
+      exit_reason: 'success',
+      agent_environment: 'harness',
+      has_agent: 'true',
+    });
+  });
+});

integ-tests/create-frameworks.test.ts

@@ -69,6 +69,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen
     telemetry.assertMetricEmitted({
       command: 'create',
       exit_reason: 'success',
+      agent_environment: 'runtime',
       agent_language: 'python',
       agent_framework: 'langchain_langgraph',
       model_provider: 'bedrock',

integ-tests/dev-server.test.ts

@@ -109,6 +109,7 @@ describe('integration: dev server', () => {
         command: 'dev',
         dev_action: 'server',
         ui_mode: 'terminal',
+        agent_environment: 'runtime',
         exit_reason: 'success',
       });
       telemetry.clearEntries();
@@ -123,6 +124,7 @@ describe('integration: dev server', () => {
         command: 'dev',
         dev_action: 'invoke',
         ui_mode: 'terminal',
+        agent_environment: 'runtime',
         exit_reason: 'success',
         agent_protocol: 'http',
       });
@@ -135,6 +137,7 @@ describe('integration: dev server', () => {
       telemetry.assertMetricEmitted({
         command: 'dev',
         dev_action: 'invoke',
+        agent_environment: 'runtime',
         exit_reason: 'failure',
       });
 
@@ -162,9 +165,51 @@ describe('integration: dev server', () => {
       telemetry.assertMetricEmitted({
         command: 'dev',
         dev_action: 'server',
+        agent_environment: 'runtime',
         exit_reason: 'failure',
       });
     },
     15000
   );
 });
+
+const isPreviewBuild = process.env.BUILD_PREVIEW === '1';
+
+describe.skipIf(!isPreviewBuild || !hasNpm || !hasGit || !hasUv)('integration: dev with harness-only project', () => {
+  const telemetry = createTelemetryHelper();
+  let projectPath: string;
+
+  beforeAll(async () => {
+    const dir = join(tmpdir(), `agentcore-dev-harness-${Date.now()}`);
+    await mkdir(dir, { recursive: true });
+
+    // Create a harness-only project
+    const createResult = await runCLI(
+      ['create', '--name', 'DevHarness', '--model-provider', 'bedrock', '--json'],
+      dir,
+      { env: telemetry.env }
+    );
+    const json = JSON.parse(createResult.stdout);
+    projectPath = json.projectPath;
+  });
+
+  afterAll(async () => {
+    telemetry.destroy();
+    if (projectPath) await rm(projectPath, { recursive: true, force: true });
+  });
+
+  // This test currently fails due to https://github.com/aws/agentcore-cli/issues/1406
+  it.skip('dev --logs on harness-only project should fail with validation error', async () => {
+    telemetry.clearEntries();
+    const result = await runCLI(['dev', '--logs', '--skip-deploy'], projectPath, { env: telemetry.env });
+
+    expect(result.exitCode).toBe(1);
+
+    telemetry.assertMetricEmitted({
+      command: 'dev',
+      dev_action: 'server',
+      agent_environment: 'harness',
+      exit_reason: 'failure',
+    });
+  });
+});

src/cli/commands/create/command.tsx

@@ -9,10 +9,11 @@ import type {
   TargetLanguage,
 } from '../../../schema';
 import { LIFECYCLE_TIMEOUT_MAX, LIFECYCLE_TIMEOUT_MIN } from '../../../schema';
+import { ANSI } from '../../constants';
 import { getErrorMessage } from '../../errors';
 import { isPreviewEnabled } from '../../feature-flags';
 import { harnessPrimitive } from '../../primitives/registry';
-import { runCliCommand } from '../../telemetry/cli-command-run.js';
+import { runCliCommand, withCommandRunTelemetry } from '../../telemetry/cli-command-run.js';
 import {
   AgentFramework,
   AgentLanguage,
@@ -139,78 +140,83 @@ async function handleCreateHarnessCLI(options: CreateOptions): Promise<void> {
   const name = options.name ?? options.projectName;
   const projectName = options.projectName ?? name;
 
-  const validation = validateCreateHarnessOptions(
+  const result = await withCommandRunTelemetry(
+    'create',
     {
-      name,
-      projectName,
-      modelProvider: options.modelProvider,
-      modelId: options.modelId,
-      apiKeyArn: options.apiKeyArn,
+      agent_environment: 'harness' as const,
+      has_agent: true,
+      model_provider: standardize(ModelProviderEnum, options.modelProvider ?? 'bedrock'),
+      memory_type: standardize(MemoryType, options.harnessMemory === false ? 'none' : 'longandshortterm'),
+      network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'),
     },
-    cwd
-  );
-  if (!validation.valid) {
-    if (options.json) {
-      console.log(JSON.stringify({ success: false, error: validation.error }));
-    } else {
-      console.error(validation.error);
-    }
-    process.exit(1);
-  }
+    async () => {
+      const validation = validateCreateHarnessOptions(
+        {
+          name,
+          projectName,
+          modelProvider: options.modelProvider,
+          modelId: options.modelId,
+          apiKeyArn: options.apiKeyArn,
+        },
+        cwd
+      );
+      if (!validation.valid) {
+        return { success: false as const, error: new ValidationError(validation.error!) };
+      }
 
-  // Progress callback
-  const green = '\x1b[32m';
-  const reset = '\x1b[0m';
-  const onProgress: ProgressCallback | undefined = options.json
-    ? undefined
-    : (step, status) => {
-        if (status === 'done') console.log(`${green}[done]${reset}  ${step}`);
-        else if (status === 'error') console.log(`\x1b[31m[error]${reset} ${step}`);
-      };
+      // Progress callback
+      const onProgress: ProgressCallback | undefined = options.json
+        ? undefined
+        : (step, status) => {
+            if (status === 'done') console.log(`${ANSI.green}[done]${ANSI.reset}  ${step}`);
+            else if (status === 'error') console.log(`${ANSI.red}[error]${ANSI.reset} ${step}`);
+          };
 
-  const provider = (
-    options.modelProvider ? normalizeHarnessModelProvider(options.modelProvider) : 'bedrock'
-  ) as HarnessModelProvider;
-  const defaultModelIds: Record<string, string> = {
-    bedrock: 'global.anthropic.claude-sonnet-4-6',
-    open_ai: 'gpt-5',
-    gemini: 'gemini-2.5-flash',
-  };
-  const modelId = options.modelId ?? defaultModelIds[provider] ?? 'global.anthropic.claude-sonnet-4-6';
-
-  const containerOption = harnessPrimitive!.parseContainerFlag(options.container);
-
-  const result = await createProjectWithHarness({
-    name: name!,
-    projectName: projectName!,
-    cwd,
-    modelProvider: provider,
-    modelId,
-    apiKeyArn: options.apiKeyArn,
-    containerUri: containerOption.containerUri,
-    dockerfilePath: containerOption.dockerfilePath,
-    skipMemory: options.harnessMemory === false,
-    maxIterations: options.maxIterations ? Number(options.maxIterations) : undefined,
-    maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined,
-    timeoutSeconds: options.timeout ? Number(options.timeout) : undefined,
-    truncationStrategy: options.truncationStrategy as 'sliding_window' | 'summarization' | undefined,
-    networkMode: options.networkMode as NetworkMode | undefined,
-    subnets: parseCommaSeparatedList(options.subnets),
-    securityGroups: parseCommaSeparatedList(options.securityGroups),
-    idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined,
-    maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined,
-    sessionStoragePath: options.sessionStorageMountPath,
-    skipGit: options.skipGit,
-    skipInstall: options.skipInstall,
-    onProgress,
-  });
+      const provider = (
+        options.modelProvider ? normalizeHarnessModelProvider(options.modelProvider) : 'bedrock'
+      ) as HarnessModelProvider;
+      const defaultModelIds: Record<string, string> = {
+        bedrock: 'global.anthropic.claude-sonnet-4-6',
+        open_ai: 'gpt-5',
+        gemini: 'gemini-2.5-flash',
+      };
+      const modelId = options.modelId ?? defaultModelIds[provider] ?? 'global.anthropic.claude-sonnet-4-6';
+
+      const containerOption = harnessPrimitive!.parseContainerFlag(options.container);
+
+      return createProjectWithHarness({
+        name: name!,
+        projectName: projectName!,
+        cwd,
+        modelProvider: provider,
+        modelId,
+        apiKeyArn: options.apiKeyArn,
+        containerUri: containerOption.containerUri,
+        dockerfilePath: containerOption.dockerfilePath,
+        skipMemory: options.harnessMemory === false,
+        maxIterations: options.maxIterations ? Number(options.maxIterations) : undefined,
+        maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined,
+        timeoutSeconds: options.timeout ? Number(options.timeout) : undefined,
+        truncationStrategy: options.truncationStrategy as 'sliding_window' | 'summarization' | undefined,
+        networkMode: options.networkMode as NetworkMode | undefined,
+        subnets: parseCommaSeparatedList(options.subnets),
+        securityGroups: parseCommaSeparatedList(options.securityGroups),
+        idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined,
+        maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined,
+        sessionStoragePath: options.sessionStorageMountPath,
+        skipGit: options.skipGit,
+        skipInstall: options.skipInstall,
+        onProgress,
+      });
+    }
+  );
 
   if (options.json) {
-    console.log(JSON.stringify(result));
+    console.log(JSON.stringify(serializeResult(result)));
   } else if (result.success) {
     printCreateHarnessSummary(projectName!, name!);
   } else {
-    console.error(result.error);
+    console.error(result.error instanceof Error ? result.error.message : 'Create failed');
   }
   process.exit(result.success ? 0 : 1);
 }
@@ -245,6 +251,7 @@ async function handleCreateCLI(options: CreateOptions): Promise<void> {
   }
 
   const knownAttrs = {
+    agent_environment: 'runtime' as const,
     agent_language: standardize(AgentLanguage, options.language),
     agent_framework: standardize(AgentFramework, options.framework),
     model_provider: standardize(ModelProviderEnum, options.modelProvider),

src/cli/commands/deploy/__tests__/utils.test.ts

@@ -16,6 +16,7 @@ describe('computeDeployAttrs', () => {
 
     expect(computeDeployAttrs(projectSpec, 'diff')).toEqual({
       runtime_count: 2,
+      harness_count: 0,
       memory_count: 1,
       credential_count: 3,
       evaluator_count: 1,
@@ -31,6 +32,7 @@ describe('computeDeployAttrs', () => {
   it('returns zeros for empty spec', () => {
     expect(computeDeployAttrs({}, 'deploy')).toEqual({
       runtime_count: 0,
+      harness_count: 0,
       memory_count: 0,
       credential_count: 0,
       evaluator_count: 0,

src/cli/commands/deploy/utils.ts

@@ -3,6 +3,7 @@ import type { DeployMode } from '../../telemetry/schemas/common-shapes';
 
 export const DEFAULT_DEPLOY_ATTRS = {
   runtime_count: 0,
+  harness_count: 0,
   memory_count: 0,
   credential_count: 0,
   evaluator_count: 0,
@@ -19,6 +20,7 @@ export function computeDeployAttrs(projectSpec: Partial<AgentCoreProjectSpec>, m
   const policyEngines = projectSpec.policyEngines ?? [];
   return {
     runtime_count: (projectSpec.runtimes ?? []).length,
+    harness_count: (projectSpec.harnesses ?? []).length,
     memory_count: (projectSpec.memories ?? []).length,
     credential_count: (projectSpec.credentials ?? []).length,
     evaluator_count: (projectSpec.evaluators ?? []).length,

src/cli/commands/dev/command.tsx

@@ -201,6 +201,7 @@ export const registerDev = (program: Command) => {
           const execResult = await withCommandRunTelemetry(
             'dev',
             {
+              agent_environment: 'runtime' as const,
               dev_action: 'exec' as const,
               ui_mode: 'terminal' as const,
               has_stream: false,
@@ -239,6 +240,7 @@ export const registerDev = (program: Command) => {
           const invokeResult = await withCommandRunTelemetry(
             'dev',
             {
+              agent_environment: 'runtime' as const,
               dev_action: 'invoke' as const,
               ui_mode: 'terminal' as const,
               has_stream: opts.stream ?? false,
@@ -301,6 +303,7 @@ export const registerDev = (program: Command) => {
         const serverResult = await withCommandRunTelemetry(
           'dev',
           {
+            agent_environment: 'runtime' as const,
             dev_action: 'server' as const,
             ui_mode: 'terminal' as const,
             has_stream: false,
@@ -353,6 +356,7 @@ export const registerDev = (program: Command) => {
             if (opts.logs) {
               // Preview: harness-only projects need deploy then print invoke instructions
               if (isPreviewEnabled() && supportedAgents.length === 0 && hasHarnesses) {
+                recorder.set({ agent_environment: 'harness' as const });
                 if (!opts.skipDeploy) {
                   await runCliDeploy();
                 }