Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 27 additions & 3 deletions src/cli/aws/agentcore-harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,34 @@ import { randomUUID } from 'node:crypto';

export type HarnessStatus = 'CREATING' | 'READY' | 'UPDATING' | 'DELETING' | 'DELETED' | 'FAILED';

export interface BedrockModelConfig {
Comment thread
jesseturner21 marked this conversation as resolved.
modelId: string;
temperature?: number;
topP?: number;
maxTokens?: number;
}

export interface OpenAiModelConfig {
modelId: string;
apiKeyArn?: string;
temperature?: number;
topP?: number;
maxTokens?: number;
}

export interface GeminiModelConfig {
modelId: string;
apiKeyArn?: string;
temperature?: number;
topP?: number;
topK?: number;
maxTokens?: number;
}

Comment thread
jesseturner21 marked this conversation as resolved.
export interface HarnessModelConfiguration {
bedrockModelConfig?: { modelId: string };
openAiModelConfig?: { modelId: string; apiKeyArn?: string };
geminiModelConfig?: { modelId: string; apiKeyArn?: string };
bedrockModelConfig?: BedrockModelConfig;
openAiModelConfig?: OpenAiModelConfig;
geminiModelConfig?: GeminiModelConfig;
}

export type HarnessSystemPrompt = { text: string }[];
Expand Down
154 changes: 154 additions & 0 deletions src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import { buildHarnessBaseOpts } from '../action.js';
import type { InvokeOptions } from '../types.js';
import { describe, expect, it } from 'vitest';

describe('buildHarnessBaseOpts', () => {
describe('preserves model inference params from harness spec when overriding model', () => {
it('bedrock: includes temperature, topP, and maxTokens', () => {
const options: InvokeOptions = { modelId: 'anthropic.claude-v3' };
const harnessSpec = {
provider: 'bedrock' as const,
modelId: 'anthropic.claude-v2',
temperature: 0.7,
topP: 0.9,
maxTokens: 500,
};

const result = buildHarnessBaseOpts(options, harnessSpec);

expect(result.model).toEqual({
bedrockModelConfig: {
modelId: 'anthropic.claude-v3',
temperature: 0.7,
topP: 0.9,
maxTokens: 500,
},
});
});

it('open_ai: includes temperature, topP, maxTokens, and apiKeyArn', () => {
const options: InvokeOptions = { modelId: 'gpt-5' };
const harnessSpec = {
provider: 'open_ai' as const,
modelId: 'gpt-4',
apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:key',
temperature: 0.5,
topP: 0.8,
maxTokens: 2048,
};

const result = buildHarnessBaseOpts(options, harnessSpec);

expect(result.model).toEqual({
openAiModelConfig: {
modelId: 'gpt-5',
apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:key',
temperature: 0.5,
topP: 0.8,
maxTokens: 2048,
},
});
});

it('gemini: includes temperature, topP, topK, maxTokens, and apiKeyArn', () => {
const options: InvokeOptions = { modelId: 'gemini-2.5-pro' };
const harnessSpec = {
provider: 'gemini' as const,
modelId: 'gemini-2.5-flash',
apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:gemini',
temperature: 0.3,
topP: 0.95,
topK: 0.5,
maxTokens: 1024,
};

const result = buildHarnessBaseOpts(options, harnessSpec);

expect(result.model).toEqual({
geminiModelConfig: {
modelId: 'gemini-2.5-pro',
apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:gemini',
temperature: 0.3,
topP: 0.95,
topK: 0.5,
maxTokens: 1024,
},
});
});
});

describe('omits undefined inference params', () => {
it('bedrock: only includes modelId when no inference params set', () => {
const options: InvokeOptions = { modelId: 'anthropic.claude-v3' };
const harnessSpec = { provider: 'bedrock' as const, modelId: 'anthropic.claude-v2' };

const result = buildHarnessBaseOpts(options, harnessSpec);

expect(result.model).toEqual({
bedrockModelConfig: { modelId: 'anthropic.claude-v3' },
});
});

it('open_ai: omits apiKeyArn and inference params when not set', () => {
const options: InvokeOptions = { modelId: 'gpt-5', modelProvider: 'open_ai' };
const harnessSpec = { provider: 'open_ai' as const, modelId: 'gpt-4' };

const result = buildHarnessBaseOpts(options, harnessSpec);

expect(result.model).toEqual({
openAiModelConfig: { modelId: 'gpt-5' },
});
});
});

describe('CLI options take precedence for apiKeyArn', () => {
it('uses CLI apiKeyArn over harness spec', () => {
const options: InvokeOptions = {
modelId: 'gpt-5',
apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:cli-key',
};
const harnessSpec = {
provider: 'open_ai' as const,
modelId: 'gpt-4',
apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:spec-key',
maxTokens: 1000,
};

const result = buildHarnessBaseOpts(options, harnessSpec);

expect(result.model!.openAiModelConfig!.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123:secret:cli-key');
expect(result.model!.openAiModelConfig!.maxTokens).toBe(1000);
});
});

describe('does not set model when no model override options provided', () => {
it('returns empty opts when no model-related options are set', () => {
const options: InvokeOptions = {};
const harnessSpec = {
provider: 'bedrock' as const,
modelId: 'anthropic.claude-v2',
maxTokens: 500,
};

const result = buildHarnessBaseOpts(options, harnessSpec);

expect(result.model).toBeUndefined();
});
});

describe('harness-level execution limits', () => {
it('forwards maxTokens, maxIterations, and timeoutSeconds from CLI options', () => {
const options: InvokeOptions = {
maxTokens: 100,
maxIterations: 10,
harnessTimeout: 30,
};

const result = buildHarnessBaseOpts(options);

expect(result.maxTokens).toBe(100);
expect(result.maxIterations).toBe(10);
expect(result.timeoutSeconds).toBe(30);
});
});
});
46 changes: 34 additions & 12 deletions src/cli/commands/invoke/action.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ConfigIO } from '../../../lib';
import type { AgentCoreProjectSpec, AwsDeploymentTargets, DeployedState } from '../../../schema';
import type { AgentCoreProjectSpec, AwsDeploymentTargets, DeployedState, HarnessModel } from '../../../schema';
import {
buildAguiRunInput,
executeBashCommand,
Expand Down Expand Up @@ -512,30 +512,52 @@ export async function handleInvoke(context: InvokeContext, options: InvokeOption
// Shared Harness Helpers
// ============================================================================

interface HarnessModel {
provider?: string;
modelId?: string;
apiKeyArn?: string;
}

function buildHarnessBaseOpts(
export function buildHarnessBaseOpts(
options: InvokeOptions,
harnessSpec?: HarnessModel
harnessSpec?: Partial<HarnessModel>
): Partial<import('../../aws/agentcore-harness').InvokeHarnessOptions> {
const baseOpts: Partial<import('../../aws/agentcore-harness').InvokeHarnessOptions> = {};
if (options.modelId || options.modelProvider || options.apiKeyArn) {
const provider = options.modelProvider ?? harnessSpec?.provider;
const modelId = options.modelId ?? harnessSpec?.modelId ?? '';
const apiKeyArn = options.apiKeyArn ?? harnessSpec?.apiKeyArn;
const temperature = harnessSpec?.temperature;
const topP = harnessSpec?.topP;
const topK = harnessSpec?.topK;
const modelMaxTokens = harnessSpec?.maxTokens;
switch (provider) {
case 'open_ai':
baseOpts.model = { openAiModelConfig: { modelId, ...(apiKeyArn && { apiKeyArn }) } };
baseOpts.model = {
openAiModelConfig: {
modelId,
...(apiKeyArn && { apiKeyArn }),
...(temperature !== undefined && { temperature }),
...(topP !== undefined && { topP }),
...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }),
},
};
break;
case 'gemini':
baseOpts.model = { geminiModelConfig: { modelId, ...(apiKeyArn && { apiKeyArn }) } };
baseOpts.model = {
geminiModelConfig: {
modelId,
...(apiKeyArn && { apiKeyArn }),
...(temperature !== undefined && { temperature }),
...(topP !== undefined && { topP }),
...(topK !== undefined && { topK }),
...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }),
},
};
break;
default:
baseOpts.model = { bedrockModelConfig: { modelId } };
baseOpts.model = {
bedrockModelConfig: {
modelId,
...(temperature !== undefined && { temperature }),
...(topP !== undefined && { topP }),
...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }),
},
};
break;
}
Comment thread
jesseturner21 marked this conversation as resolved.
}
Expand Down
2 changes: 1 addition & 1 deletion src/schema/schemas/agentcore-project.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ export type { ABTestMode, TargetRef, GatewayFilter, PerVariantOnlineEvaluationCo
export { ABTestModeSchema, TargetRefSchema, GatewayFilterSchema } from './primitives/ab-test';
export type { HttpGatewayTarget } from './primitives/http-gateway';
export { HttpGatewayTargetSchema } from './primitives/http-gateway';
export type { HarnessGatewayOutboundAuth, HarnessSpec, HarnessModelProvider } from './primitives/harness';
export type { HarnessGatewayOutboundAuth, HarnessModel, HarnessSpec, HarnessModelProvider } from './primitives/harness';
export {
GatewayOAuthGrantTypeSchema,
HarnessGatewayOutboundAuthSchema,
Expand Down
Loading