Skip to content

Commit eac09a5

Browse files
committed
add sim auto model to custom provider json
1 parent a7f022a commit eac09a5

6 files changed

Lines changed: 137 additions & 8 deletions

File tree

apps/sim/blocks/blocks/agent.test.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ describe('AgentBlock', () => {
5353
})
5454
)
5555
expect(AgentBlock.inputs.customModelConfig?.schema).toBeDefined()
56+
expect(AgentBlock.inputs.customModelConfig?.schema?.properties.provider.enum).toContain('sim')
5657

5758
const tools = AgentBlock.subBlocks.find((subBlock) => subBlock.id === 'tools')
5859
const responseFormat = AgentBlock.subBlocks.find((subBlock) => subBlock.id === 'responseFormat')
@@ -79,6 +80,20 @@ describe('AgentBlock', () => {
7980
).toBe('fireworks')
8081
})
8182

83+
it('serializes custom Sim Auto with the same fallback provider shape as first-class Auto', () => {
84+
const toolFunction = AgentBlock.tools.config?.tool
85+
86+
expect(
87+
toolFunction?.({
88+
model: CUSTOM_MODEL_ID,
89+
customModelConfig: {
90+
provider: 'sim',
91+
model: 'sim-auto',
92+
},
93+
} as never)
94+
).toBe('anthropic')
95+
})
96+
8297
describe('tools.config.params function', () => {
8398
it('should pass through params when no tools array is provided', () => {
8499
const params = {

apps/sim/blocks/blocks/agent.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ Return ONLY the JSON array.`,
152152
language: 'json',
153153
placeholder: 'Enter custom provider and model configuration...',
154154
description:
155-
'Provider, model, credentials, and provider-specific generation settings. Tools, response format, skills, files, prompts, and memory remain configured separately.',
155+
'Provider, model, credentials, and provider-specific generation settings. Use provider "sim" with model "sim-auto" for automatic routing. Tools, response format, skills, files, prompts, and memory remain configured separately.',
156156
defaultValue: CUSTOM_MODEL_CONFIG_DEFAULT,
157157
superUserOnly: true,
158158
required: {
@@ -542,7 +542,13 @@ Return ONLY the JSON array.`,
542542
throw new Error('No model selected')
543543
}
544544
if (isCustomModel(model)) {
545-
return parseCustomModelConfig(params.customModelConfig).provider
545+
const customConfig = parseCustomModelConfig(params.customModelConfig)
546+
if (customConfig.provider !== 'sim') return customConfig.provider
547+
548+
// Like the first-class Auto option below, custom Sim Auto resolves to
549+
// a concrete provider only at execution time. Serialization stores a
550+
// valid fallback provider shape that the Agent handler never uses.
551+
return getBaseModelProviders()['claude-sonnet-5']
546552
}
547553
// sim-auto resolves to a concrete pool model at execution time, where
548554
// the agent handler derives the provider from the resolved model and
@@ -631,7 +637,7 @@ Return ONLY the JSON array.`,
631637
customModelConfig: {
632638
type: 'json',
633639
description:
634-
'Custom provider/model execution configuration. Tools, response format, skills, files, prompts, and memory are separate Agent inputs.',
640+
'Custom provider/model execution configuration. Use provider "sim" with model "sim-auto" for automatic routing. Tools, response format, skills, files, prompts, and memory are separate Agent inputs.',
635641
schema: CUSTOM_MODEL_CONFIG_JSON_SCHEMA,
636642
},
637643
apiKey: { type: 'string', description: 'Provider API key' },

apps/sim/executor/handlers/agent/agent-handler.test.ts

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,35 @@ describe('AgentBlockHandler', () => {
391391
expect(result.cost).toEqual({ input: 0.001, output: 0.002, total: 0.003 })
392392
})
393393

394+
it('runs Sim Auto from the custom contract through catalog capability handling', async () => {
395+
mockContext.userId = 'super-user'
396+
397+
const result = (await handler.execute(mockContext, mockBlock, {
398+
model: CUSTOM_MODEL_ID,
399+
customModelConfig: {
400+
provider: 'sim',
401+
model: SIM_AUTO_MODEL_ID,
402+
parameters: { reasoningEffort: 'high', temperature: 0.2 },
403+
credentials: { mode: 'auto' },
404+
},
405+
userPrompt: 'Route this automatically',
406+
})) as { model: string }
407+
408+
expect(mockVerifyEffectiveSuperUser).toHaveBeenCalledWith('super-user')
409+
expect(mockExecuteProviderRequest).toHaveBeenCalledWith(
410+
'mock-provider',
411+
expect.objectContaining({
412+
model: AGENT.DEFAULT_MODEL,
413+
capabilityPolicy: 'catalog',
414+
credentialMode: 'auto',
415+
reasoningEffort: 'high',
416+
temperature: 0.2,
417+
}),
418+
expect.anything()
419+
)
420+
expect(result.model).toBe(SIM_AUTO_MODEL_ID)
421+
})
422+
394423
/** Reaches the private signal builder; routing depends on nothing else. */
395424
const buildAutoRoutingSignalsFor = (inputs: Record<string, unknown>) =>
396425
(

apps/sim/executor/handlers/agent/agent-handler.ts

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,12 @@ export class AgentBlockHandler implements BlockHandler {
146146
customModelConfig.credentials.mode === 'explicit'
147147
? customModelConfig.credentials.apiKey
148148
: undefined
149-
} else if (isAutoModel(configuredModel)) {
149+
}
150+
151+
if (
152+
isAutoModel(configuredModel) ||
153+
(customModelConfig !== undefined && customModelConfig.provider === 'sim')
154+
) {
150155
autoRouting = await resolveAutoModel({
151156
ctx,
152157
blockId: block.id,
@@ -169,7 +174,10 @@ export class AgentBlockHandler implements BlockHandler {
169174
.join('\n\n')
170175
}
171176

172-
const providerId = customModelConfig?.provider ?? getProviderFromModel(model)
177+
const providerId =
178+
customModelConfig && customModelConfig.provider !== 'sim'
179+
? customModelConfig.provider
180+
: getProviderFromModel(model)
173181
await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx, providerId)
174182
const formattedTools = await this.formatTools(
175183
ctx,
@@ -1224,7 +1232,10 @@ export class AgentBlockHandler implements BlockHandler {
12241232
verbosity: inputs.verbosity,
12251233
thinkingLevel: inputs.thinkingLevel,
12261234
promptCaching: inputs.promptCaching === true,
1227-
capabilityPolicy: customModelConfig ? ('passthrough' as const) : ('catalog' as const),
1235+
capabilityPolicy:
1236+
customModelConfig && customModelConfig.provider !== 'sim'
1237+
? ('passthrough' as const)
1238+
: ('catalog' as const),
12281239
credentialMode: customModelConfig?.credentials.mode,
12291240
providerOptions: customModelConfig?.providerOptions,
12301241
previousInteractionId: inputs.previousInteractionId,

apps/sim/providers/custom-model.test.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,47 @@ describe('custom model config', () => {
4949
}
5050
})
5151

52+
it('supports Sim Auto through the custom contract', () => {
53+
expect(
54+
parseCustomModelConfig({
55+
provider: 'auto',
56+
model: 'sim-auto',
57+
parameters: { reasoningEffort: 'high', temperature: 0.2 },
58+
credentials: { mode: 'auto' },
59+
})
60+
).toMatchObject({
61+
provider: 'sim',
62+
model: 'sim-auto',
63+
parameters: { reasoningEffort: 'high', temperature: 0.2 },
64+
credentials: { mode: 'auto' },
65+
})
66+
})
67+
68+
it('keeps Sim Auto provider-independent', () => {
69+
expect(() =>
70+
parseCustomModelConfig({
71+
provider: 'sim',
72+
model: 'gpt-5.6-terra',
73+
})
74+
).toThrow('model must be "sim-auto"')
75+
76+
expect(() =>
77+
parseCustomModelConfig({
78+
provider: 'sim',
79+
model: 'sim-auto',
80+
credentials: { mode: 'explicit', apiKey: 'sk-secret' },
81+
})
82+
).toThrow('credentials.mode must be "auto"')
83+
84+
expect(() =>
85+
parseCustomModelConfig({
86+
provider: 'sim',
87+
model: 'sim-auto',
88+
providerOptions: { service_tier: 'priority' },
89+
})
90+
).toThrow('providerOptions must be empty for Sim Auto')
91+
})
92+
5293
it('rejects reserved provider option overrides', () => {
5394
expect(() =>
5495
parseCustomModelConfig({

apps/sim/providers/custom-model.ts

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
1+
import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models'
12
import type { ProviderId } from '@/providers/types'
23

34
/** Stored in the Agent block's `model` field when Super User custom routing is active. */
45
export const CUSTOM_MODEL_ID = 'sim-custom'
56

6-
export const CUSTOM_MODEL_PROVIDERS = [
7+
export const CUSTOM_DIRECT_MODEL_PROVIDERS = [
78
'openai',
89
'anthropic',
910
'google',
1011
'fireworks',
1112
'xai',
1213
] as const satisfies readonly ProviderId[]
1314

15+
export const CUSTOM_MODEL_PROVIDERS = [...CUSTOM_DIRECT_MODEL_PROVIDERS, 'sim'] as const
16+
1417
export type CustomModelProvider = (typeof CUSTOM_MODEL_PROVIDERS)[number]
1518
export type CustomCredentialMode = 'auto' | 'explicit'
1619

@@ -45,6 +48,9 @@ const PROVIDER_ALIASES: Record<string, CustomModelProvider> = {
4548
fireworks: 'fireworks',
4649
xai: 'xai',
4750
grok: 'xai',
51+
sim: 'sim',
52+
auto: 'sim',
53+
'sim-auto': 'sim',
4854
}
4955

5056
const TOP_LEVEL_KEYS = new Set([
@@ -247,6 +253,23 @@ export function parseCustomModelConfig(value: unknown): CustomModelConfig {
247253
providerOptions: parseProviderOptions(parsed.providerOptions),
248254
}
249255

256+
if (config.provider === 'sim') {
257+
if (!isAutoModel(config.model)) {
258+
throw new Error(
259+
`customModelConfig.model must be "${SIM_AUTO_MODEL_ID}" when provider is "sim"`
260+
)
261+
}
262+
config.model = SIM_AUTO_MODEL_ID
263+
if (config.credentials.mode !== 'auto') {
264+
throw new Error('customModelConfig.credentials.mode must be "auto" for Sim Auto')
265+
}
266+
if (Object.keys(config.providerOptions).length > 0) {
267+
throw new Error(
268+
'customModelConfig.providerOptions must be empty for Sim Auto because the routed provider is dynamic'
269+
)
270+
}
271+
}
272+
250273
validateCustomModelParameterSupport(config)
251274
return config
252275
}
@@ -256,6 +279,10 @@ export function validateCustomModelParameterSupport(config: CustomModelConfig):
256279
const { provider, parameters } = config
257280
const hasValue = (value: unknown) => value !== undefined && value !== null && value !== 'auto'
258281

282+
// Sim Auto validates parameters against the concrete routed model through
283+
// the ordinary catalog capability policy after routing.
284+
if (provider === 'sim') return
285+
259286
if (hasValue(parameters.verbosity) && provider !== 'openai') {
260287
throw new Error(`customModelConfig.parameters.verbosity is not supported for ${provider}`)
261288
}
@@ -324,7 +351,7 @@ export const CUSTOM_MODEL_CONFIG_JSON_SCHEMA: Record<string, any> = {
324351
type: 'string',
325352
enum: [...CUSTOM_MODEL_PROVIDERS],
326353
description:
327-
'Provider adapter. Aliases gemini, claude, and grok are normalized on execution.',
354+
'Provider adapter. Use sim with model sim-auto for dynamic routing. Aliases gemini, claude, grok, and auto are normalized on execution.',
328355
},
329356
model: {
330357
type: 'string',

0 commit comments

Comments
 (0)