Skip to content

Commit 1e3f324

Browse files
committed
add sim auto model to custom provider json
1 parent 5b31897 commit 1e3f324

6 files changed

Lines changed: 137 additions & 8 deletions

File tree

apps/sim/blocks/blocks/agent.test.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ describe('AgentBlock', () => {
5353
})
5454
)
5555
expect(AgentBlock.inputs.customModelConfig?.schema).toBeDefined()
56+
expect(AgentBlock.inputs.customModelConfig?.schema?.properties.provider.enum).toContain('sim')
5657

5758
const tools = AgentBlock.subBlocks.find((subBlock) => subBlock.id === 'tools')
5859
const responseFormat = AgentBlock.subBlocks.find((subBlock) => subBlock.id === 'responseFormat')
@@ -79,6 +80,20 @@ describe('AgentBlock', () => {
7980
).toBe('fireworks')
8081
})
8182

83+
it('serializes custom Sim Auto with the same fallback provider shape as first-class Auto', () => {
84+
const toolFunction = AgentBlock.tools.config?.tool
85+
86+
expect(
87+
toolFunction?.({
88+
model: CUSTOM_MODEL_ID,
89+
customModelConfig: {
90+
provider: 'sim',
91+
model: 'sim-auto',
92+
},
93+
} as never)
94+
).toBe('anthropic')
95+
})
96+
8297
describe('tools.config.params function', () => {
8398
it('should pass through params when no tools array is provided', () => {
8499
const params = {

apps/sim/blocks/blocks/agent.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ Return ONLY the JSON array.`,
151151
language: 'json',
152152
placeholder: 'Enter custom provider and model configuration...',
153153
description:
154-
'Provider, model, credentials, and provider-specific generation settings. Tools, response format, skills, files, prompts, and memory remain configured separately.',
154+
'Provider, model, credentials, and provider-specific generation settings. Use provider "sim" with model "sim-auto" for automatic routing. Tools, response format, skills, files, prompts, and memory remain configured separately.',
155155
defaultValue: CUSTOM_MODEL_CONFIG_DEFAULT,
156156
superUserOnly: true,
157157
required: {
@@ -550,7 +550,13 @@ Return ONLY the JSON array.`,
550550
throw new Error('No model selected')
551551
}
552552
if (isCustomModel(model)) {
553-
return parseCustomModelConfig(params.customModelConfig).provider
553+
const customConfig = parseCustomModelConfig(params.customModelConfig)
554+
if (customConfig.provider !== 'sim') return customConfig.provider
555+
556+
// Like the first-class Auto option below, custom Sim Auto resolves to
557+
// a concrete provider only at execution time. Serialization stores a
558+
// valid fallback provider shape that the Agent handler never uses.
559+
return getBaseModelProviders()['claude-sonnet-5']
554560
}
555561
// sim-auto resolves to a concrete pool model at execution time, where
556562
// the agent handler derives the provider from the resolved model and
@@ -639,7 +645,7 @@ Return ONLY the JSON array.`,
639645
customModelConfig: {
640646
type: 'json',
641647
description:
642-
'Custom provider/model execution configuration. Tools, response format, skills, files, prompts, and memory are separate Agent inputs.',
648+
'Custom provider/model execution configuration. Use provider "sim" with model "sim-auto" for automatic routing. Tools, response format, skills, files, prompts, and memory are separate Agent inputs.',
643649
schema: CUSTOM_MODEL_CONFIG_JSON_SCHEMA,
644650
},
645651
apiKey: { type: 'string', description: 'Provider API key' },

apps/sim/executor/handlers/agent/agent-handler.test.ts

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,35 @@ describe('AgentBlockHandler', () => {
386386
expect(result.cost).toEqual({ input: 0.001, output: 0.002, total: 0.003 })
387387
})
388388

389+
it('runs Sim Auto from the custom contract through catalog capability handling', async () => {
390+
mockContext.userId = 'super-user'
391+
392+
const result = (await handler.execute(mockContext, mockBlock, {
393+
model: CUSTOM_MODEL_ID,
394+
customModelConfig: {
395+
provider: 'sim',
396+
model: SIM_AUTO_MODEL_ID,
397+
parameters: { reasoningEffort: 'high', temperature: 0.2 },
398+
credentials: { mode: 'auto' },
399+
},
400+
userPrompt: 'Route this automatically',
401+
})) as { model: string }
402+
403+
expect(mockVerifyEffectiveSuperUser).toHaveBeenCalledWith('super-user')
404+
expect(mockExecuteProviderRequest).toHaveBeenCalledWith(
405+
'mock-provider',
406+
expect.objectContaining({
407+
model: AGENT.DEFAULT_MODEL,
408+
capabilityPolicy: 'catalog',
409+
credentialMode: 'auto',
410+
reasoningEffort: 'high',
411+
temperature: 0.2,
412+
}),
413+
expect.anything()
414+
)
415+
expect(result.model).toBe(SIM_AUTO_MODEL_ID)
416+
})
417+
389418
/** Reaches the private signal builder; routing depends on nothing else. */
390419
const buildAutoRoutingSignalsFor = (inputs: Record<string, unknown>) =>
391420
(

apps/sim/executor/handlers/agent/agent-handler.ts

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,12 @@ export class AgentBlockHandler implements BlockHandler {
125125
customModelConfig.credentials.mode === 'explicit'
126126
? customModelConfig.credentials.apiKey
127127
: undefined
128-
} else if (isAutoModel(configuredModel)) {
128+
}
129+
130+
if (
131+
isAutoModel(configuredModel) ||
132+
(customModelConfig !== undefined && customModelConfig.provider === 'sim')
133+
) {
129134
autoRouting = await resolveAutoModel({
130135
ctx,
131136
blockId: block.id,
@@ -148,7 +153,10 @@ export class AgentBlockHandler implements BlockHandler {
148153
.join('\n\n')
149154
}
150155

151-
const providerId = customModelConfig?.provider ?? getProviderFromModel(model)
156+
const providerId =
157+
customModelConfig && customModelConfig.provider !== 'sim'
158+
? customModelConfig.provider
159+
: getProviderFromModel(model)
152160
await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx, providerId)
153161
const formattedTools = await this.formatTools(
154162
ctx,
@@ -1179,7 +1187,10 @@ export class AgentBlockHandler implements BlockHandler {
11791187
verbosity: inputs.verbosity,
11801188
thinkingLevel: inputs.thinkingLevel,
11811189
promptCaching: inputs.promptCaching === true,
1182-
capabilityPolicy: customModelConfig ? ('passthrough' as const) : ('catalog' as const),
1190+
capabilityPolicy:
1191+
customModelConfig && customModelConfig.provider !== 'sim'
1192+
? ('passthrough' as const)
1193+
: ('catalog' as const),
11831194
credentialMode: customModelConfig?.credentials.mode,
11841195
providerOptions: customModelConfig?.providerOptions,
11851196
previousInteractionId: inputs.previousInteractionId,

apps/sim/providers/custom-model.test.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,47 @@ describe('custom model config', () => {
4949
}
5050
})
5151

52+
it('supports Sim Auto through the custom contract', () => {
53+
expect(
54+
parseCustomModelConfig({
55+
provider: 'auto',
56+
model: 'sim-auto',
57+
parameters: { reasoningEffort: 'high', temperature: 0.2 },
58+
credentials: { mode: 'auto' },
59+
})
60+
).toMatchObject({
61+
provider: 'sim',
62+
model: 'sim-auto',
63+
parameters: { reasoningEffort: 'high', temperature: 0.2 },
64+
credentials: { mode: 'auto' },
65+
})
66+
})
67+
68+
it('keeps Sim Auto provider-independent', () => {
69+
expect(() =>
70+
parseCustomModelConfig({
71+
provider: 'sim',
72+
model: 'gpt-5.6-terra',
73+
})
74+
).toThrow('model must be "sim-auto"')
75+
76+
expect(() =>
77+
parseCustomModelConfig({
78+
provider: 'sim',
79+
model: 'sim-auto',
80+
credentials: { mode: 'explicit', apiKey: 'sk-secret' },
81+
})
82+
).toThrow('credentials.mode must be "auto"')
83+
84+
expect(() =>
85+
parseCustomModelConfig({
86+
provider: 'sim',
87+
model: 'sim-auto',
88+
providerOptions: { service_tier: 'priority' },
89+
})
90+
).toThrow('providerOptions must be empty for Sim Auto')
91+
})
92+
5293
it('rejects reserved provider option overrides', () => {
5394
expect(() =>
5495
parseCustomModelConfig({

apps/sim/providers/custom-model.ts

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
1+
import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models'
12
import type { ProviderId } from '@/providers/types'
23

34
/** Stored in the Agent block's `model` field when Super User custom routing is active. */
45
export const CUSTOM_MODEL_ID = 'sim-custom'
56

6-
export const CUSTOM_MODEL_PROVIDERS = [
7+
export const CUSTOM_DIRECT_MODEL_PROVIDERS = [
78
'openai',
89
'anthropic',
910
'google',
1011
'fireworks',
1112
'xai',
1213
] as const satisfies readonly ProviderId[]
1314

15+
export const CUSTOM_MODEL_PROVIDERS = [...CUSTOM_DIRECT_MODEL_PROVIDERS, 'sim'] as const
16+
1417
export type CustomModelProvider = (typeof CUSTOM_MODEL_PROVIDERS)[number]
1518
export type CustomCredentialMode = 'auto' | 'explicit'
1619

@@ -45,6 +48,9 @@ const PROVIDER_ALIASES: Record<string, CustomModelProvider> = {
4548
fireworks: 'fireworks',
4649
xai: 'xai',
4750
grok: 'xai',
51+
sim: 'sim',
52+
auto: 'sim',
53+
'sim-auto': 'sim',
4854
}
4955

5056
const TOP_LEVEL_KEYS = new Set([
@@ -247,6 +253,23 @@ export function parseCustomModelConfig(value: unknown): CustomModelConfig {
247253
providerOptions: parseProviderOptions(parsed.providerOptions),
248254
}
249255

256+
if (config.provider === 'sim') {
257+
if (!isAutoModel(config.model)) {
258+
throw new Error(
259+
`customModelConfig.model must be "${SIM_AUTO_MODEL_ID}" when provider is "sim"`
260+
)
261+
}
262+
config.model = SIM_AUTO_MODEL_ID
263+
if (config.credentials.mode !== 'auto') {
264+
throw new Error('customModelConfig.credentials.mode must be "auto" for Sim Auto')
265+
}
266+
if (Object.keys(config.providerOptions).length > 0) {
267+
throw new Error(
268+
'customModelConfig.providerOptions must be empty for Sim Auto because the routed provider is dynamic'
269+
)
270+
}
271+
}
272+
250273
validateCustomModelParameterSupport(config)
251274
return config
252275
}
@@ -256,6 +279,10 @@ export function validateCustomModelParameterSupport(config: CustomModelConfig):
256279
const { provider, parameters } = config
257280
const hasValue = (value: unknown) => value !== undefined && value !== null && value !== 'auto'
258281

282+
// Sim Auto validates parameters against the concrete routed model through
283+
// the ordinary catalog capability policy after routing.
284+
if (provider === 'sim') return
285+
259286
if (hasValue(parameters.verbosity) && provider !== 'openai') {
260287
throw new Error(`customModelConfig.parameters.verbosity is not supported for ${provider}`)
261288
}
@@ -324,7 +351,7 @@ export const CUSTOM_MODEL_CONFIG_JSON_SCHEMA: Record<string, any> = {
324351
type: 'string',
325352
enum: [...CUSTOM_MODEL_PROVIDERS],
326353
description:
327-
'Provider adapter. Aliases gemini, claude, and grok are normalized on execution.',
354+
'Provider adapter. Use sim with model sim-auto for dynamic routing. Aliases gemini, claude, grok, and auto are normalized on execution.',
328355
},
329356
model: {
330357
type: 'string',

0 commit comments

Comments
 (0)