From f783d1054bd8ce3490ee767d9527a18698dfdd35 Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Mon, 22 Jun 2026 15:19:43 -0400 Subject: [PATCH 1/2] test(integ): expand harness config-shape coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an integration suite covering the harness config surface the existing tests miss — all local (no AWS), exercising CLI flag → action → validate → spec-write wiring that schema unit tests don't reach: - model: lite_llm (+api-base/additional-params), bedrock api-format, plus api-format provider-gating rejections (open_ai/gemini) - tools (`add tool`): remote_mcp, code_interpreter, gateway (awsIam), inline_function, plus required-config rejections - skills (`add skill`): path / s3 / git (+auth credential) - memory modes (gated): managed + strategies, existing-by-name + tuning, plus by-arn / no-ref / gated-off rejections - truncation: summarization - VPC network-mode validation (no deploy) - CUSTOM_JWT inbound authorizer + missing-config rejection Cases share one project (single `create` spawn) and use it.each tables for rejection groups to keep the suite fast. Pure schema refinements already covered by harness.test.ts unit tests are intentionally not duplicated. All 35 tests in the file pass. --- integ-tests/add-remove-harness.test.ts | 353 +++++++++++++++++++++++++ 1 file changed, 353 insertions(+) diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts index c72a33197..ea4654355 100644 --- a/integ-tests/add-remove-harness.test.ts +++ b/integ-tests/add-remove-harness.test.ts @@ -8,6 +8,12 @@ async function readHarnessSpec(projectPath: string, harnessName: string) { return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8')); } +// Placeholder ARNs/keys for config-shape tests (never resolved against AWS). +const OPENAI_KEY_ARN = 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key'; +const GEMINI_KEY_ARN = 'arn:aws:secretsmanager:us-east-1:123456789012:secret:gemini-key'; +const GATEWAY_ARN = 'arn:aws:bedrock-agentcore:us-east-1:123456789012:gateway/abc123'; +const MEMORY_ARN = 'arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/abc123'; + describe('integration: harness add/remove lifecycle', () => { let project: TestProject; const harnessName = 'TestHarness'; @@ -216,3 +222,350 @@ describe('integration: create project with harness', () => { expect(harness).toBeTruthy(); }); }); + +// ============================================================================ +// Config-shape coverage — model, tools, skills, memory modes, truncation, auth. +// +// All cases below share ONE project (one `create` spawn). Each `runCLI` is a +// real process spawn (~1.3s), so cases are kept to behaviors that the schema +// unit tests (src/schema/.../__tests__/harness.test.ts) don't already cover at +// the parse layer — i.e. CLI flag → action → spec-write wiring. Pure schema +// refinements (top-k/api-base provider gating, skill URI scheme) live in unit +// tests and are intentionally NOT duplicated here. +// ============================================================================ + +describe('integration: harness config shape', () => { + let project: TestProject; + const gatedEnv = { ENABLE_GATED_FEATURES: '1' }; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + // Resources referenced by name in the cases below. + await runCLI(['add', 'memory', '--name', 'SharedMem', '--json'], project.projectPath); + await runCLI( + ['add', 'credential', '--name', 'gitCred', '--api-key', 'test-token-integ', '--json'], + project.projectPath + ); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + describe('model configuration', () => { + it('adds a lite_llm harness with --api-base and --additional-params', async () => { + const name = 'LiteLlmHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--model-provider', + 'lite_llm', + '--model-id', + 'together_ai/meta-llama/Llama-3-70b', + '--api-base', + 'https://api.together.xyz/v1', + '--additional-params', + '{"drop_params":true}', + '--no-memory', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.model.provider).toBe('lite_llm'); + expect(spec.model.apiBase).toBe('https://api.together.xyz/v1'); + expect(spec.model.additionalParams).toEqual({ drop_params: true }); + }); + + it('adds a bedrock harness with --api-format converse_stream', async () => { + const name = 'BedrockFmt'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--model-provider', + 'bedrock', + '--api-format', + 'converse_stream', + '--no-memory', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.model.apiFormat).toBe('converse_stream'); + }); + + // --api-format provider gating is enforced by the CLI validator (not just the schema), + // so these exercise the wiring rather than duplicating a parse-layer unit test. + it.each([ + { + label: 'converse_stream for open_ai', + args: ['--model-provider', 'open_ai', '--model-id', 'gpt-5', '--api-key-arn', OPENAI_KEY_ARN], + apiFormat: 'converse_stream', + }, + { + label: 'any --api-format for gemini', + args: ['--model-provider', 'gemini', '--model-id', 'gemini-2.0-flash', '--api-key-arn', GEMINI_KEY_ARN], + apiFormat: 'responses', + }, + ])('rejects $label', async ({ args, apiFormat }) => { + const result = await runCLI( + ['add', 'harness', '--name', 'BadFmt', ...args, '--api-format', apiFormat, '--no-memory', '--json'], + project.projectPath + ); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('tools (`add tool`)', () => { + beforeAll(async () => { + await runCLI(['add', 'harness', '--name', 'ToolHarness', '--no-memory', '--json'], project.projectPath); + }); + + it('adds remote_mcp, code_interpreter, gateway, and inline_function tools', async () => { + const cmds: string[][] = [ + ['--type', 'remote_mcp', '--name', 'myMcp', '--url', 'https://mcp.example.com/sse'], + ['--type', 'agentcore_code_interpreter', '--name', 'codeRunner'], + ['--type', 'agentcore_gateway', '--name', 'gw', '--gateway-arn', GATEWAY_ARN, '--outbound-auth', 'awsIam'], + [ + '--type', + 'inline_function', + '--name', + 'lookup', + '--description', + 'Look up a value', + '--input-schema', + '{"type":"object","properties":{"q":{"type":"string"}}}', + ], + ]; + for (const extra of cmds) { + const result = await runCLI( + ['add', 'tool', '--harness', 'ToolHarness', ...extra, '--json'], + project.projectPath + ); + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + } + + const spec = await readHarnessSpec(project.projectPath, 'ToolHarness'); + const byName = (n: string) => spec.tools.find((t: { name: string }) => t.name === n); + expect(byName('myMcp').config.remoteMcp.url).toBe('https://mcp.example.com/sse'); + expect(byName('codeRunner').type).toBe('agentcore_code_interpreter'); + expect(byName('gw').config.agentCoreGateway.gatewayArn).toBe(GATEWAY_ARN); + expect(byName('gw').config.agentCoreGateway.outboundAuth).toEqual({ awsIam: {} }); + expect(byName('lookup').config.inlineFunction.description).toBe('Look up a value'); + expect(byName('lookup').config.inlineFunction.inputSchema).toEqual({ + type: 'object', + properties: { q: { type: 'string' } }, + }); + }); + + it.each([ + { label: 'remote_mcp without --url', args: ['--type', 'remote_mcp', '--name', 'noUrl'] }, + { + label: 'inline_function without --input-schema', + args: ['--type', 'inline_function', '--name', 'noSchema', '--description', 'Missing schema'], + }, + { + label: 'agentcore_gateway without --gateway-arn or --gateway', + args: ['--type', 'agentcore_gateway', '--name', 'noArn'], + }, + ])('rejects $label', async ({ args }) => { + const result = await runCLI(['add', 'tool', '--harness', 'ToolHarness', ...args, '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('skills (`add skill`)', () => { + beforeAll(async () => { + await runCLI(['add', 'harness', '--name', 'SkillHarness', '--no-memory', '--json'], project.projectPath); + }); + + it('adds path, s3, and git (+auth) skills', async () => { + const cmds: string[][] = [ + ['--path', 'skills/my-skill'], + ['--s3', 's3://my-bucket/skills/pack'], + [ + '--git', + 'https://github.com/example/skills.git', + '--git-path', + 'pack', + '--credential', + 'gitCred', + '--username', + 'git-user', + ], + ]; + for (const extra of cmds) { + const result = await runCLI( + ['add', 'skill', '--harness', 'SkillHarness', ...extra, '--json'], + project.projectPath + ); + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + } + + const spec = await readHarnessSpec(project.projectPath, 'SkillHarness'); + expect(spec.skills.some((s: { path?: string }) => s.path === 'skills/my-skill')).toBe(true); + expect(spec.skills.some((s: { s3Uri?: string }) => s.s3Uri === 's3://my-bucket/skills/pack')).toBe(true); + const git = spec.skills.find((s: { gitUrl?: string }) => s.gitUrl === 'https://github.com/example/skills.git'); + expect(git.path).toBe('pack'); + expect(git.auth).toEqual({ credentialName: 'gitCred', username: 'git-user' }); + }); + }); + + describe('memory modes (gated)', () => { + it('adds a managed-memory harness with explicit strategies', async () => { + const name = 'ManagedMemHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--memory-mode', + 'managed', + '--memory-strategies', + 'SEMANTIC,EPISODIC', + '--json', + ], + project.projectPath, + { env: gatedEnv } + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.memory.mode).toBe('managed'); + expect(spec.memory.strategies).toEqual(['SEMANTIC', 'EPISODIC']); + }); + + it('adds an existing-memory harness referencing a sibling by name with tuning', async () => { + const name = 'ExistingMemHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--memory-mode', + 'existing', + '--memory-name', + 'SharedMem', + '--memory-messages-count', + '20', + '--memory-top-k', + '5', + '--json', + ], + project.projectPath, + { env: gatedEnv } + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.memory.mode).toBe('existing'); + expect(spec.memory.name).toBe('SharedMem'); + expect(spec.memory.messagesCount).toBe(20); + expect(spec.memory.retrievalConfig.topK).toBe(5); + }); + + it.each([ + { + label: 'retrievalConfig tuning when memory is referenced by ARN', + args: ['--memory-mode', 'existing', '--memory-arn', MEMORY_ARN, '--memory-top-k', '5'], + env: gatedEnv, + }, + { + label: '--memory-mode existing without a memory reference', + args: ['--memory-mode', 'existing'], + env: gatedEnv, + }, + { + label: '--memory-mode when the gated feature is disabled', + args: ['--memory-mode', 'managed'], + env: undefined, + }, + ])('rejects $label', async ({ args, env }) => { + const result = await runCLI( + ['add', 'harness', '--name', 'BadMem', ...args, '--json'], + project.projectPath, + env ? { env } : {} + ); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('truncation', () => { + it('adds a harness with summarization truncation', async () => { + const name = 'SummHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--truncation-strategy', 'summarization', '--no-memory', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.truncation.strategy).toBe('summarization'); + }); + }); + + describe('VPC network-mode validation (no deploy)', () => { + it.each([ + { + label: '--subnets / --security-groups without --network-mode VPC', + args: ['--subnets', 'subnet-123', '--security-groups', 'sg-123'], + }, + { label: '--network-mode VPC without subnets/security groups', args: ['--network-mode', 'VPC'] }, + ])('rejects $label', async ({ args }) => { + const result = await runCLI( + ['add', 'harness', '--name', 'BadVpc', ...args, '--no-memory', '--json'], + project.projectPath + ); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('CUSTOM_JWT inbound authorizer', () => { + it('adds a harness with a CUSTOM_JWT authorizer', async () => { + const name = 'JwtHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--authorizer-type', + 'CUSTOM_JWT', + '--discovery-url', + 'https://example.com/.well-known/openid-configuration', + '--allowed-audience', + 'my-audience', + '--no-memory', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.authorizerType).toBe('CUSTOM_JWT'); + expect(spec.authorizerConfiguration?.customJwtAuthorizer).toBeDefined(); + }); + + it('rejects CUSTOM_JWT without authorizer configuration', async () => { + const result = await runCLI( + ['add', 'harness', '--name', 'JwtNoConfig', '--authorizer-type', 'CUSTOM_JWT', '--no-memory', '--json'], + project.projectPath + ); + expect(result.exitCode).not.toBe(0); + }); + }); +}); From 26bb9e99dacaa8fac8c2082ac7d092c24a2de34c Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Mon, 22 Jun 2026 19:01:38 -0400 Subject: [PATCH 2/2] test(integ): make gated-off harness case immune to host ENABLE_GATED_FEATURES The 'rejects --memory-mode when gated off' case relied on the gate being unset in the environment. cleanSpawnEnv inherits the host process.env and does not strip ENABLE_GATED_FEATURES, so a shell/CI with it exported as '1' would flip the case: the CLI accepts the flag, exits 0, and the rejection assertion fails for an environment-dependent reason. Force the gate off explicitly via env { ENABLE_GATED_FEATURES: '' } ('' is off since isGatedFeaturesEnabled checks === '1'). Verified the block passes both with the gate unset and with ENABLE_GATED_FEATURES=1 exported. --- integ-tests/add-remove-harness.test.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts index ea4654355..7893c5e8a 100644 --- a/integ-tests/add-remove-harness.test.ts +++ b/integ-tests/add-remove-harness.test.ts @@ -490,15 +490,17 @@ describe('integration: harness config shape', () => { }, { label: '--memory-mode when the gated feature is disabled', + // Force the gate OFF explicitly: cleanSpawnEnv inherits the host process.env, so a + // developer/CI shell with ENABLE_GATED_FEATURES=1 exported would otherwise flip this + // case (the CLI would accept --memory-mode and exit 0). An empty string is "off" + // because isGatedFeaturesEnabled() checks `=== '1'`. args: ['--memory-mode', 'managed'], - env: undefined, + env: { ENABLE_GATED_FEATURES: '' }, }, ])('rejects $label', async ({ args, env }) => { - const result = await runCLI( - ['add', 'harness', '--name', 'BadMem', ...args, '--json'], - project.projectPath, - env ? { env } : {} - ); + const result = await runCLI(['add', 'harness', '--name', 'BadMem', ...args, '--json'], project.projectPath, { + env, + }); expect(result.exitCode).not.toBe(0); }); });