diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts index c72a33197..7893c5e8a 100644 --- a/integ-tests/add-remove-harness.test.ts +++ b/integ-tests/add-remove-harness.test.ts @@ -8,6 +8,12 @@ async function readHarnessSpec(projectPath: string, harnessName: string) { return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8')); } +// Placeholder ARNs/keys for config-shape tests (never resolved against AWS). +const OPENAI_KEY_ARN = 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key'; +const GEMINI_KEY_ARN = 'arn:aws:secretsmanager:us-east-1:123456789012:secret:gemini-key'; +const GATEWAY_ARN = 'arn:aws:bedrock-agentcore:us-east-1:123456789012:gateway/abc123'; +const MEMORY_ARN = 'arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/abc123'; + describe('integration: harness add/remove lifecycle', () => { let project: TestProject; const harnessName = 'TestHarness'; @@ -216,3 +222,352 @@ describe('integration: create project with harness', () => { expect(harness).toBeTruthy(); }); }); + +// ============================================================================ +// Config-shape coverage — model, tools, skills, memory modes, truncation, auth. +// +// All cases below share ONE project (one `create` spawn). Each `runCLI` is a +// real process spawn (~1.3s), so cases are kept to behaviors that the schema +// unit tests (src/schema/.../__tests__/harness.test.ts) don't already cover at +// the parse layer — i.e. CLI flag → action → spec-write wiring. Pure schema +// refinements (top-k/api-base provider gating, skill URI scheme) live in unit +// tests and are intentionally NOT duplicated here. +// ============================================================================ + +describe('integration: harness config shape', () => { + let project: TestProject; + const gatedEnv = { ENABLE_GATED_FEATURES: '1' }; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + // Resources referenced by name in the cases below. + await runCLI(['add', 'memory', '--name', 'SharedMem', '--json'], project.projectPath); + await runCLI( + ['add', 'credential', '--name', 'gitCred', '--api-key', 'test-token-integ', '--json'], + project.projectPath + ); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + describe('model configuration', () => { + it('adds a lite_llm harness with --api-base and --additional-params', async () => { + const name = 'LiteLlmHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--model-provider', + 'lite_llm', + '--model-id', + 'together_ai/meta-llama/Llama-3-70b', + '--api-base', + 'https://api.together.xyz/v1', + '--additional-params', + '{"drop_params":true}', + '--no-memory', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.model.provider).toBe('lite_llm'); + expect(spec.model.apiBase).toBe('https://api.together.xyz/v1'); + expect(spec.model.additionalParams).toEqual({ drop_params: true }); + }); + + it('adds a bedrock harness with --api-format converse_stream', async () => { + const name = 'BedrockFmt'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--model-provider', + 'bedrock', + '--api-format', + 'converse_stream', + '--no-memory', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.model.apiFormat).toBe('converse_stream'); + }); + + // --api-format provider gating is enforced by the CLI validator (not just the schema), + // so these exercise the wiring rather than duplicating a parse-layer unit test. + it.each([ + { + label: 'converse_stream for open_ai', + args: ['--model-provider', 'open_ai', '--model-id', 'gpt-5', '--api-key-arn', OPENAI_KEY_ARN], + apiFormat: 'converse_stream', + }, + { + label: 'any --api-format for gemini', + args: ['--model-provider', 'gemini', '--model-id', 'gemini-2.0-flash', '--api-key-arn', GEMINI_KEY_ARN], + apiFormat: 'responses', + }, + ])('rejects $label', async ({ args, apiFormat }) => { + const result = await runCLI( + ['add', 'harness', '--name', 'BadFmt', ...args, '--api-format', apiFormat, '--no-memory', '--json'], + project.projectPath + ); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('tools (`add tool`)', () => { + beforeAll(async () => { + await runCLI(['add', 'harness', '--name', 'ToolHarness', '--no-memory', '--json'], project.projectPath); + }); + + it('adds remote_mcp, code_interpreter, gateway, and inline_function tools', async () => { + const cmds: string[][] = [ + ['--type', 'remote_mcp', '--name', 'myMcp', '--url', 'https://mcp.example.com/sse'], + ['--type', 'agentcore_code_interpreter', '--name', 'codeRunner'], + ['--type', 'agentcore_gateway', '--name', 'gw', '--gateway-arn', GATEWAY_ARN, '--outbound-auth', 'awsIam'], + [ + '--type', + 'inline_function', + '--name', + 'lookup', + '--description', + 'Look up a value', + '--input-schema', + '{"type":"object","properties":{"q":{"type":"string"}}}', + ], + ]; + for (const extra of cmds) { + const result = await runCLI( + ['add', 'tool', '--harness', 'ToolHarness', ...extra, '--json'], + project.projectPath + ); + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + } + + const spec = await readHarnessSpec(project.projectPath, 'ToolHarness'); + const byName = (n: string) => spec.tools.find((t: { name: string }) => t.name === n); + expect(byName('myMcp').config.remoteMcp.url).toBe('https://mcp.example.com/sse'); + expect(byName('codeRunner').type).toBe('agentcore_code_interpreter'); + expect(byName('gw').config.agentCoreGateway.gatewayArn).toBe(GATEWAY_ARN); + expect(byName('gw').config.agentCoreGateway.outboundAuth).toEqual({ awsIam: {} }); + expect(byName('lookup').config.inlineFunction.description).toBe('Look up a value'); + expect(byName('lookup').config.inlineFunction.inputSchema).toEqual({ + type: 'object', + properties: { q: { type: 'string' } }, + }); + }); + + it.each([ + { label: 'remote_mcp without --url', args: ['--type', 'remote_mcp', '--name', 'noUrl'] }, + { + label: 'inline_function without --input-schema', + args: ['--type', 'inline_function', '--name', 'noSchema', '--description', 'Missing schema'], + }, + { + label: 'agentcore_gateway without --gateway-arn or --gateway', + args: ['--type', 'agentcore_gateway', '--name', 'noArn'], + }, + ])('rejects $label', async ({ args }) => { + const result = await runCLI(['add', 'tool', '--harness', 'ToolHarness', ...args, '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('skills (`add skill`)', () => { + beforeAll(async () => { + await runCLI(['add', 'harness', '--name', 'SkillHarness', '--no-memory', '--json'], project.projectPath); + }); + + it('adds path, s3, and git (+auth) skills', async () => { + const cmds: string[][] = [ + ['--path', 'skills/my-skill'], + ['--s3', 's3://my-bucket/skills/pack'], + [ + '--git', + 'https://github.com/example/skills.git', + '--git-path', + 'pack', + '--credential', + 'gitCred', + '--username', + 'git-user', + ], + ]; + for (const extra of cmds) { + const result = await runCLI( + ['add', 'skill', '--harness', 'SkillHarness', ...extra, '--json'], + project.projectPath + ); + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + } + + const spec = await readHarnessSpec(project.projectPath, 'SkillHarness'); + expect(spec.skills.some((s: { path?: string }) => s.path === 'skills/my-skill')).toBe(true); + expect(spec.skills.some((s: { s3Uri?: string }) => s.s3Uri === 's3://my-bucket/skills/pack')).toBe(true); + const git = spec.skills.find((s: { gitUrl?: string }) => s.gitUrl === 'https://github.com/example/skills.git'); + expect(git.path).toBe('pack'); + expect(git.auth).toEqual({ credentialName: 'gitCred', username: 'git-user' }); + }); + }); + + describe('memory modes (gated)', () => { + it('adds a managed-memory harness with explicit strategies', async () => { + const name = 'ManagedMemHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--memory-mode', + 'managed', + '--memory-strategies', + 'SEMANTIC,EPISODIC', + '--json', + ], + project.projectPath, + { env: gatedEnv } + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.memory.mode).toBe('managed'); + expect(spec.memory.strategies).toEqual(['SEMANTIC', 'EPISODIC']); + }); + + it('adds an existing-memory harness referencing a sibling by name with tuning', async () => { + const name = 'ExistingMemHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--memory-mode', + 'existing', + '--memory-name', + 'SharedMem', + '--memory-messages-count', + '20', + '--memory-top-k', + '5', + '--json', + ], + project.projectPath, + { env: gatedEnv } + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.memory.mode).toBe('existing'); + expect(spec.memory.name).toBe('SharedMem'); + expect(spec.memory.messagesCount).toBe(20); + expect(spec.memory.retrievalConfig.topK).toBe(5); + }); + + it.each([ + { + label: 'retrievalConfig tuning when memory is referenced by ARN', + args: ['--memory-mode', 'existing', '--memory-arn', MEMORY_ARN, '--memory-top-k', '5'], + env: gatedEnv, + }, + { + label: '--memory-mode existing without a memory reference', + args: ['--memory-mode', 'existing'], + env: gatedEnv, + }, + { + label: '--memory-mode when the gated feature is disabled', + // Force the gate OFF explicitly: cleanSpawnEnv inherits the host process.env, so a + // developer/CI shell with ENABLE_GATED_FEATURES=1 exported would otherwise flip this + // case (the CLI would accept --memory-mode and exit 0). An empty string is "off" + // because isGatedFeaturesEnabled() checks `=== '1'`. + args: ['--memory-mode', 'managed'], + env: { ENABLE_GATED_FEATURES: '' }, + }, + ])('rejects $label', async ({ args, env }) => { + const result = await runCLI(['add', 'harness', '--name', 'BadMem', ...args, '--json'], project.projectPath, { + env, + }); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('truncation', () => { + it('adds a harness with summarization truncation', async () => { + const name = 'SummHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--truncation-strategy', 'summarization', '--no-memory', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.truncation.strategy).toBe('summarization'); + }); + }); + + describe('VPC network-mode validation (no deploy)', () => { + it.each([ + { + label: '--subnets / --security-groups without --network-mode VPC', + args: ['--subnets', 'subnet-123', '--security-groups', 'sg-123'], + }, + { label: '--network-mode VPC without subnets/security groups', args: ['--network-mode', 'VPC'] }, + ])('rejects $label', async ({ args }) => { + const result = await runCLI( + ['add', 'harness', '--name', 'BadVpc', ...args, '--no-memory', '--json'], + project.projectPath + ); + expect(result.exitCode).not.toBe(0); + }); + }); + + describe('CUSTOM_JWT inbound authorizer', () => { + it('adds a harness with a CUSTOM_JWT authorizer', async () => { + const name = 'JwtHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--authorizer-type', + 'CUSTOM_JWT', + '--discovery-url', + 'https://example.com/.well-known/openid-configuration', + '--allowed-audience', + 'my-audience', + '--no-memory', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.authorizerType).toBe('CUSTOM_JWT'); + expect(spec.authorizerConfiguration?.customJwtAuthorizer).toBeDefined(); + }); + + it('rejects CUSTOM_JWT without authorizer configuration', async () => { + const result = await runCLI( + ['add', 'harness', '--name', 'JwtNoConfig', '--authorizer-type', 'CUSTOM_JWT', '--no-memory', '--json'], + project.projectPath + ); + expect(result.exitCode).not.toBe(0); + }); + }); +});