From cd5d0e67938591bf07957b85904a6677459f7c7c Mon Sep 17 00:00:00 2001 From: Dabbler Date: Mon, 10 Aug 2026 19:47:11 +0800 Subject: [PATCH] feat: add gemini interactions native tools --- .../conversation/model_option_policy_test.go | 46 +++++++- .../internal/application/settings/service.go | 66 +++++++++-- .../application/settings/service_seed_test.go | 61 ++++++++++ backend/internal/infra/config/config.go | 27 +++-- .../internal/infra/llm/gemini_interactions.go | 8 +- .../infra/llm/gemini_interactions_test.go | 55 ++++++++- backend/internal/shared/nativetool/catalog.go | 40 ++++++- .../shared/nativetool/catalog_test.go | 13 +++ .../conversation/admin-conversation.tsx | 2 +- .../models/models-capabilities-config.tsx | 110 +++++++++++++++--- .../models/models-capabilities-presets.tsx | 84 ++++++++++++- .../admin/model/conversation-settings.ts | 29 ++--- .../components/sections/chat-model-config.tsx | 3 + .../chat/hooks/use-chat-model-options.ts | 13 ++- .../i18n/messages/en-US/admin-models.json | 4 +- frontend/i18n/messages/en-US/chat.json | 6 + .../i18n/messages/zh-CN/admin-models.json | 4 +- frontend/i18n/messages/zh-CN/chat.json | 6 + 18 files changed, 506 insertions(+), 71 deletions(-) diff --git a/backend/internal/application/conversation/model_option_policy_test.go b/backend/internal/application/conversation/model_option_policy_test.go index ef8fb7b74..d8a10375d 100644 --- a/backend/internal/application/conversation/model_option_policy_test.go +++ b/backend/internal/application/conversation/model_option_policy_test.go @@ -678,6 +678,10 @@ func TestFilterModelOptionsGeminiPolicyKeyMatchesGoogleAdapter(t *testing.T) { "temperature": 0.4, "responseMimeType": "application/json", "candidateCount": 3, + "thinkingConfig": map[string]interface{}{ + "includeThoughts": true, + "thinkingLevel": "high", + }, }, "tools": []interface{}{ map[string]interface{}{"type": "google_search"}, @@ -696,6 +700,10 @@ func TestFilterModelOptionsGeminiPolicyKeyMatchesGoogleAdapter(t *testing.T) { if _, ok := generationConfig["candidateCount"]; ok { t.Fatalf("expected unlisted gemini option removed, got %#v", generationConfig) } + thinkingConfig, ok := generationConfig["thinkingConfig"].(map[string]interface{}) + if !ok || thinkingConfig["includeThoughts"] != true || thinkingConfig["thinkingLevel"] != "high" { + t.Fatalf("expected Gemini thinking options to pass, got %#v", generationConfig) + } tools := filtered["tools"].([]map[string]interface{}) if len(tools) != 1 { t.Fatalf("expected Gemini google_search tool, got %#v", tools) @@ -972,8 +980,10 @@ func TestFilterModelOptionsGeminiInteractionsAllowsVideoParams(t *testing.T) { "delivery": "b64_json", }, "generation_config": map[string]interface{}{ - "temperature": 0.3, - "thinking_level": "low", + "temperature": 0.3, + "thinking_level": "low", + "thinking_summaries": "auto", + "max_output_tokens": 1024, "video_config": map[string]interface{}{ "task": "image_to_video", }, @@ -998,7 +1008,10 @@ func TestFilterModelOptionsGeminiInteractionsAllowsVideoParams(t *testing.T) { t.Fatalf("expected Gemini generation_config to pass, got %#v", filtered) } videoConfig, ok := generationConfig["video_config"].(map[string]interface{}) - if generationConfig["temperature"] != 0.3 || generationConfig["thinking_level"] != "low" { + if generationConfig["temperature"] != 0.3 || + generationConfig["thinking_level"] != "low" || + generationConfig["thinking_summaries"] != "auto" || + generationConfig["max_output_tokens"] != 1024 { t.Fatalf("expected Gemini generation config fields to pass, got %#v", generationConfig) } if !ok || videoConfig["task"] != "image_to_video" { @@ -1011,6 +1024,33 @@ func TestFilterModelOptionsGeminiInteractionsAllowsVideoParams(t *testing.T) { } } +func TestFilterModelOptionsGeminiInteractionsPreservesConfiguredNativeTools(t *testing.T) { + filtered := filterModelOptions(map[string]interface{}{ + "tools": []interface{}{ + map[string]interface{}{"type": "google_search"}, + map[string]interface{}{"type": "code_execution"}, + map[string]interface{}{"type": "url_context"}, + map[string]interface{}{"type": "external_function", "name": "not_allowed"}, + }, + }, llm.AdapterGeminiInteractions, modelOptionPolicyConfig{ + Mode: modelOptionPolicyAllowlist, + AllowedPathsJSON: config.DefaultModelOptionAllowedPathsJSON(), + DeniedPathsJSON: config.DefaultModelOptionDeniedPathsJSON(), + ModelCapabilitiesJSON: `{"nativeToolKeys":["google.google_search","google.code_execution","google.url_context"]}`, + }) + + tools, ok := filtered["tools"].([]map[string]interface{}) + if !ok || len(tools) != 3 { + t.Fatalf("expected three configured Gemini Interactions tools, got %#v", filtered["tools"]) + } + wantTypes := []string{"google_search", "code_execution", "url_context"} + for index, wantType := range wantTypes { + if tools[index]["type"] != wantType { + t.Fatalf("tool %d type = %#v, want %q", index, tools[index]["type"], wantType) + } + } +} + func TestFilterModelOptionsGeminiInteractionsAllowsCamelCaseVideoConfig(t *testing.T) { filtered := filterModelOptions(map[string]interface{}{ "generationConfig": map[string]interface{}{ diff --git a/backend/internal/application/settings/service.go b/backend/internal/application/settings/service.go index 87574d209..f78e48bbc 100644 --- a/backend/internal/application/settings/service.go +++ b/backend/internal/application/settings/service.go @@ -198,16 +198,68 @@ func isLegacyDefaultModelOptionAllowedPaths(value string) bool { if err := json.Unmarshal([]byte(strings.TrimSpace(value)), ¤t); err != nil { return false } - previousDefault := map[string][]string{} - if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &previousDefault); err != nil { + latestDefault := map[string][]string{} + if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &latestDefault); err != nil { return false } - delete(previousDefault, "xai_video") - if sameStringSliceMap(current, previousDefault) { - return true + previousGenerateContentDefault := cloneStringSliceMap(latestDefault) + previousGenerateContentDefault["gemini_generate_content"] = removeStringValue( + removeStringValue( + previousGenerateContentDefault["gemini_generate_content"], + "generationConfig.thinkingConfig.includeThoughts", + ), + "generationConfig.thinkingConfig.thinkingLevel", + ) + previousInteractionsDefault := cloneStringSliceMap(latestDefault) + previousInteractionsDefault["gemini_interactions"] = removeStringValue( + previousInteractionsDefault["gemini_interactions"], + "generation_config.thinking_summaries", + ) + previousCombinedDefault := cloneStringSliceMap(previousGenerateContentDefault) + previousCombinedDefault["gemini_interactions"] = removeStringValue( + previousCombinedDefault["gemini_interactions"], + "generation_config.thinking_summaries", + ) + previousDefaults := []map[string][]string{ + previousGenerateContentDefault, + previousInteractionsDefault, + previousCombinedDefault, + } + for _, previousDefault := range previousDefaults { + if sameStringSliceMap(current, previousDefault) { + return true + } + } + olderDefaults := append([]map[string][]string{cloneStringSliceMap(latestDefault)}, previousDefaults...) + for _, olderDefault := range olderDefaults { + delete(olderDefault, "xai_video") + if sameStringSliceMap(current, olderDefault) { + return true + } + olderDefault["xai_responses"] = []string{"reasoning.effort"} + if sameStringSliceMap(current, olderDefault) { + return true + } + } + return false +} + +func cloneStringSliceMap(value map[string][]string) map[string][]string { + result := make(map[string][]string, len(value)) + for key, items := range value { + result[key] = append([]string(nil), items...) } - previousDefault["xai_responses"] = []string{"reasoning.effort"} - return sameStringSliceMap(current, previousDefault) + return result +} + +func removeStringValue(values []string, target string) []string { + result := make([]string, 0, len(values)) + for _, value := range values { + if value != target { + result = append(result, value) + } + } + return result } func sameStringSliceMap(left map[string][]string, right map[string][]string) bool { diff --git a/backend/internal/application/settings/service_seed_test.go b/backend/internal/application/settings/service_seed_test.go index 22662adbd..a68eea0cc 100644 --- a/backend/internal/application/settings/service_seed_test.go +++ b/backend/internal/application/settings/service_seed_test.go @@ -184,6 +184,67 @@ func TestSeedAddsXAIVideoToPreviousDefaultModelOptionAllowedPaths(t *testing.T) } } +func TestSeedAddsGeminiThinkingSummariesToPreviousDefaultModelOptionAllowedPaths(t *testing.T) { + previousDefault := map[string][]string{} + if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &previousDefault); err != nil { + t.Fatalf("decode current model option defaults: %v", err) + } + previousDefault["gemini_interactions"] = removeStringValue( + previousDefault["gemini_interactions"], + "generation_config.thinking_summaries", + ) + previousJSON, err := json.Marshal(previousDefault) + if err != nil { + t.Fatalf("encode previous model option defaults: %v", err) + } + repo := newSettingsSeedRepo(domainsettings.SystemSetting{ + Namespace: "chat", + Key: "model_option_allowed_paths", + Value: string(previousJSON), + ValueType: "json", + }) + service := NewService(repo, "") + + if err := service.Seed(context.Background(), config.Config{}); err != nil { + t.Fatalf("seed settings: %v", err) + } + if got := repo.items["chat:model_option_allowed_paths"].Value; got != config.DefaultModelOptionAllowedPathsJSON() { + t.Fatalf("expected Gemini thinking summaries default to be added, got %q", got) + } +} + +func TestSeedAddsGeminiGenerateContentThinkingPathsToPreviousDefaultModelOptionAllowedPaths(t *testing.T) { + previousDefault := map[string][]string{} + if err := json.Unmarshal([]byte(config.DefaultModelOptionAllowedPathsJSON()), &previousDefault); err != nil { + t.Fatalf("decode current model option defaults: %v", err) + } + previousDefault["gemini_generate_content"] = removeStringValue( + removeStringValue( + previousDefault["gemini_generate_content"], + "generationConfig.thinkingConfig.includeThoughts", + ), + "generationConfig.thinkingConfig.thinkingLevel", + ) + previousJSON, err := json.Marshal(previousDefault) + if err != nil { + t.Fatalf("encode previous model option defaults: %v", err) + } + repo := newSettingsSeedRepo(domainsettings.SystemSetting{ + Namespace: "chat", + Key: "model_option_allowed_paths", + Value: string(previousJSON), + ValueType: "json", + }) + service := NewService(repo, "") + + if err := service.Seed(context.Background(), config.Config{}); err != nil { + t.Fatalf("seed settings: %v", err) + } + if got := repo.items["chat:model_option_allowed_paths"].Value; got != config.DefaultModelOptionAllowedPathsJSON() { + t.Fatalf("expected Gemini Generate Content thinking defaults to be added, got %q", got) + } +} + func TestSeedKeepsCustomModelOptionAllowedPaths(t *testing.T) { custom := `{"default":["temperature"],"xai_responses":["reasoning.effort"]}` repo := newSettingsSeedRepo(domainsettings.SystemSetting{ diff --git a/backend/internal/infra/config/config.go b/backend/internal/infra/config/config.go index 209f7a570..6406fb820 100644 --- a/backend/internal/infra/config/config.go +++ b/backend/internal/infra/config/config.go @@ -113,6 +113,20 @@ func DefaultModelOptionAllowedPathsJSON() string { "size", "user" ], + "anthropic_messages": [ + "speed", + "top_k", + "thinking.type", + "thinking.budget_tokens" + ], + "gemini_generate_content": [ + "generationConfig.temperature", + "generationConfig.topP", + "generationConfig.maxOutputTokens", + "generationConfig.responseMimeType", + "generationConfig.thinkingConfig.includeThoughts", + "generationConfig.thinkingConfig.thinkingLevel" + ], "google_image_generation": [ "generationConfig.responseModalities", "generationConfig.imageConfig.aspectRatio", @@ -123,6 +137,7 @@ func DefaultModelOptionAllowedPathsJSON() string { "generation_config.top_p", "generation_config.max_output_tokens", "generation_config.thinking_level", + "generation_config.thinking_summaries", "response_format.type", "response_format.aspect_ratio", "response_format.image_size", @@ -134,12 +149,6 @@ func DefaultModelOptionAllowedPathsJSON() string { "generationConfig.videoConfig.task", "generation_config.video_config.task" ], - "anthropic_messages": [ - "speed", - "top_k", - "thinking.type", - "thinking.budget_tokens" - ], "xai_responses": [ "reasoning.effort", "min_p", @@ -163,12 +172,6 @@ func DefaultModelOptionAllowedPathsJSON() string { "aspect_ratio", "duration", "resolution" - ], - "gemini_generate_content": [ - "generationConfig.temperature", - "generationConfig.topP", - "generationConfig.maxOutputTokens", - "generationConfig.responseMimeType" ] }` } diff --git a/backend/internal/infra/llm/gemini_interactions.go b/backend/internal/infra/llm/gemini_interactions.go index cdfe64329..64f579288 100644 --- a/backend/internal/infra/llm/gemini_interactions.go +++ b/backend/internal/infra/llm/gemini_interactions.go @@ -161,8 +161,12 @@ func buildGeminiInteractionRequestBody(route RouteConfig, input GenerateInput) ( if previousID := strings.TrimSpace(input.PreviousResponseID); previousID != "" { payload["previous_interaction_id"] = previousID } - if tools := buildGeminiInteractionTools(input.Tools); len(tools) > 0 && !input.DisableTools { - payload["tools"] = tools + providerTools, toolDefinitions, toolsEnabled, err := toolDeclarationsForInput(input) + if err != nil { + return nil, err + } + if toolsEnabled { + appendToolDeclarations(payload, providerTools, buildGeminiInteractionTools(toolDefinitions)) } applyProviderOptions(payload, input.Options, geminiInteractionsProtectedProviderOptionKeys()...) return payload, nil diff --git a/backend/internal/infra/llm/gemini_interactions_test.go b/backend/internal/infra/llm/gemini_interactions_test.go index 7c885daaf..856377299 100644 --- a/backend/internal/infra/llm/gemini_interactions_test.go +++ b/backend/internal/infra/llm/gemini_interactions_test.go @@ -145,7 +145,8 @@ func TestBuildGeminiInteractionRequestBodySupportsUniversalOptionsAndTools(t *te "max_output_tokens": 512, "thinking_level": "low", "generation_config": map[string]interface{}{ - "thinkingLevel": "high", + "thinkingLevel": "high", + "thinking_summaries": "auto", }, }, }) @@ -164,7 +165,7 @@ func TestBuildGeminiInteractionRequestBodySupportsUniversalOptionsAndTools(t *te t.Fatalf("unexpected image response_format: %#v", imageFormat) } config, ok := payload["generation_config"].(map[string]interface{}) - if !ok || config["temperature"] != 0.4 || config["top_p"] != 0.9 || config["max_output_tokens"] != 512 || config["thinking_level"] != "low" { + if !ok || config["temperature"] != 0.4 || config["top_p"] != 0.9 || config["max_output_tokens"] != 512 || config["thinking_level"] != "low" || config["thinking_summaries"] != "auto" { t.Fatalf("unexpected generation_config: %#v", payload["generation_config"]) } tools, ok := payload["tools"].([]map[string]interface{}) @@ -193,6 +194,56 @@ func TestBuildGeminiInteractionRequestBodySupportsUniversalOptionsAndTools(t *te } } +func TestBuildGeminiInteractionRequestBodyMergesNativeAndFunctionTools(t *testing.T) { + input := GenerateInput{ + Messages: []Message{{Role: "user", Content: "Research and calculate."}}, + Options: map[string]interface{}{ + "tools": []interface{}{ + map[string]interface{}{"type": "google_search"}, + map[string]interface{}{"type": "code_execution"}, + map[string]interface{}{"type": "url_context"}, + }, + }, + Tools: []ToolDefinition{{ + Name: "get_weather", + Description: "Gets weather.", + InputSchema: json.RawMessage(`{"type":"object"}`), + }}, + } + payload, err := buildGeminiInteractionRequestBody(RouteConfig{ + Endpoint: EndpointInteractions, + UpstreamModel: "gemini-3.5-flash", + }, input) + if err != nil { + t.Fatalf("build Gemini interaction request body: %v", err) + } + tools, ok := payload["tools"].([]map[string]interface{}) + if !ok || len(tools) != 4 { + t.Fatalf("expected three native tools and one function, got %#v", payload["tools"]) + } + wantTypes := []string{"google_search", "code_execution", "url_context", "function"} + for index, wantType := range wantTypes { + if tools[index]["type"] != wantType { + t.Fatalf("tool %d type = %#v, want %q", index, tools[index]["type"], wantType) + } + } + if tools[3]["name"] != "get_weather" { + t.Fatalf("expected function tool to be preserved, got %#v", tools[3]) + } + + input.DisableTools = true + disabledPayload, err := buildGeminiInteractionRequestBody(RouteConfig{ + Endpoint: EndpointInteractions, + UpstreamModel: "gemini-3.5-flash", + }, input) + if err != nil { + t.Fatalf("build disabled-tools Gemini interaction request body: %v", err) + } + if _, exists := disabledPayload["tools"]; exists { + t.Fatalf("expected DisableTools to remove native and function tools, got %#v", disabledPayload["tools"]) + } +} + func TestBuildGeminiInteractionToolsPreservesJSONSchemaReferences(t *testing.T) { payload, err := buildGeminiInteractionRequestBody(RouteConfig{ Endpoint: EndpointInteractions, diff --git a/backend/internal/shared/nativetool/catalog.go b/backend/internal/shared/nativetool/catalog.go index 507252c28..deeab0f72 100644 --- a/backend/internal/shared/nativetool/catalog.go +++ b/backend/internal/shared/nativetool/catalog.go @@ -66,6 +66,7 @@ var protocolOrder = []string{ "anthropic_messages", "xai_responses", "gemini_generate_content", + "gemini_interactions", "google_image_generation", } @@ -374,6 +375,43 @@ var definitions = []Definition{ UsageAliases: []string{"url_context"}, rawTypeFieldKeys: []string{"url_context", "urlContext"}, }, + { + Protocol: "gemini_interactions", + Provider: "Google", + Type: "google_search", + Key: "google.google_search", + Label: "Google Search", + Description: "Google hosted search grounding tool.", + Payload: map[string]interface{}{"type": "google_search"}, + DefaultEnabled: true, + PriceLabel: "notMetered", + UsageAliases: []string{"google_search"}, + }, + { + Protocol: "gemini_interactions", + Provider: "Google", + Type: "code_execution", + Key: "google.code_execution", + Label: "Code Execution", + Description: "Google hosted code execution tool.", + Payload: map[string]interface{}{"type": "code_execution"}, + DefaultEnabled: true, + PriceLabel: "notMetered", + RiskLevel: "high", + UsageAliases: []string{"code_execution"}, + }, + { + Protocol: "gemini_interactions", + Provider: "Google", + Type: "url_context", + Key: "google.url_context", + Label: "URL Context", + Description: "Google hosted URL context tool.", + Payload: map[string]interface{}{"type": "url_context"}, + DefaultEnabled: true, + PriceLabel: "notMetered", + UsageAliases: []string{"url_context"}, + }, } var usagePricesByKey = map[string]UsagePrice{ @@ -898,7 +936,7 @@ func UsagePricingKey(protocol string, toolName string) (string, bool) { case "file_search", "collection_search", "collections_search": return "xai.collections_search", true } - case "gemini_generate_content", "google_image_generation": + case "gemini_generate_content", "gemini_interactions", "google_image_generation": switch tool { case "google_search": return "google.google_search", true diff --git a/backend/internal/shared/nativetool/catalog_test.go b/backend/internal/shared/nativetool/catalog_test.go index 10eccd90c..fd19ca8a8 100644 --- a/backend/internal/shared/nativetool/catalog_test.go +++ b/backend/internal/shared/nativetool/catalog_test.go @@ -62,6 +62,15 @@ func TestPayloadFromOptionPreservesToolParametersAndFixesIdentity(t *testing.T) t.Fatalf("expected %s definition payload to preserve empty object, got %#v", item, definition.Payload) } } + for _, item := range []string{"google_search", "code_execution", "url_context"} { + definition, ok := Find("gemini_interactions", item) + if !ok { + t.Fatalf("expected Gemini Interactions %s definition", item) + } + if definition.Payload["type"] != item { + t.Fatalf("expected Gemini Interactions %s type payload, got %#v", item, definition.Payload) + } + } for _, item := range []string{"code_execution", "url_context"} { _, payload, ok = PayloadFromOption("gemini_generate_content", map[string]interface{}{ @@ -126,6 +135,10 @@ func TestUsagePricingKeyMapsObservedToolUsage(t *testing.T) { if !ok || key != "google.url_context" { t.Fatalf("expected Google URL context price key, got key=%q ok=%v", key, ok) } + key, ok = UsagePricingKey("gemini_interactions", "code_execution") + if !ok || key != "google.code_execution" { + t.Fatalf("expected Gemini Interactions code execution price key, got key=%q ok=%v", key, ok) + } } func TestPricingOverridesApplyToDisplayAndUsagePricing(t *testing.T) { diff --git a/frontend/features/admin/components/sections/conversation/admin-conversation.tsx b/frontend/features/admin/components/sections/conversation/admin-conversation.tsx index fede411d2..29908055e 100644 --- a/frontend/features/admin/components/sections/conversation/admin-conversation.tsx +++ b/frontend/features/admin/components/sections/conversation/admin-conversation.tsx @@ -337,7 +337,7 @@ generationConfig.safetySettings.threshold`}

{t("guide.protocolTitle")}

{t("guide.protocolDescription")}

- {["default", "openai_chat_completions", "openrouter_chat_completions", "openai_responses", "openrouter_responses", "openai_image_generations", "openai_image_edits", "google_image_generation", "gemini_interactions", "xai_image", "xai_image_edits", "xai_video", "anthropic_messages", "xai_responses", "gemini_generate_content"].map((item) => ( + {MODEL_OPTION_POLICY_PROTOCOLS.map((item) => ( {item} ))}
diff --git a/frontend/features/admin/components/sections/models/models-capabilities-config.tsx b/frontend/features/admin/components/sections/models/models-capabilities-config.tsx index 57e4ba6c8..33af86afe 100644 --- a/frontend/features/admin/components/sections/models/models-capabilities-config.tsx +++ b/frontend/features/admin/components/sections/models/models-capabilities-config.tsx @@ -520,10 +520,27 @@ function sortNativeToolOptionsByRoute( return leftMatched ? -1 : 1; } const providerOrder = left.provider.localeCompare(right.provider); - return providerOrder || left.label.localeCompare(right.label) || left.toolKey.localeCompare(right.toolKey) || left.type.localeCompare(right.type); + return providerOrder + || left.label.localeCompare(right.label) + || left.toolKey.localeCompare(right.toolKey) + || left.type.localeCompare(right.type) + || left.protocols.join(",").localeCompare(right.protocols.join(",")); }); } +function nativeToolPayloadSignature(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((item) => nativeToolPayloadSignature(item)).join(",")}]`; + } + if (isPlainJSONObject(value)) { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${nativeToolPayloadSignature(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? String(value); +} + function nativeToolOptionsFromCatalog( nativeTools: NativeToolDefinition[], routeProtocols: string[] = [], @@ -533,12 +550,29 @@ function nativeToolOptionsFromCatalog( nativeTools.forEach((tool) => { const toolKey = tool.toolKey.trim(); const type = tool.type.trim(); - const id = nativeToolOptionID(toolKey, type); - const existing = options.get(id); + const protocol = canonicalNativeToolProtocol(tool.protocol); + const payload = tool.payload ?? {}; + const payloadSignature = nativeToolPayloadSignature(payload); + const existing = Array.from(options.values()).find((option) => + option.toolKey === toolKey + && option.type === type + && nativeToolPayloadSignature(option.payload) === payloadSignature, + ); if (existing) { - existing.protocols = Array.from(new Set([...existing.protocols, tool.protocol].filter(Boolean))); + const existingMatchesRoute = existing.protocols.some((protocol) => + routeProtocolSet.has(resolveModelOptionPolicyProtocol(protocol)), + ); + const toolMatchesRoute = routeProtocolSet.has(resolveModelOptionPolicyProtocol(protocol)); + if (toolMatchesRoute && !existingMatchesRoute) { + existing.provider = tool.provider || "Provider"; + existing.label = tool.label || tool.type || tool.toolKey; + existing.description = tool.description || tool.type || tool.toolKey; + existing.payload = payload; + } + existing.protocols = Array.from(new Set([...existing.protocols, protocol].filter(Boolean))); return; } + const id = `${nativeToolOptionID(toolKey, type, [protocol])}:${payloadSignature}`; options.set(id, { id, toolKey, @@ -546,15 +580,19 @@ function nativeToolOptionsFromCatalog( label: tool.label || tool.type || tool.toolKey, description: tool.description || tool.type || tool.toolKey, type, - payload: tool.payload ?? {}, - protocols: [tool.protocol].filter(Boolean), + payload, + protocols: [protocol].filter(Boolean), }); }); return sortNativeToolOptionsByRoute(Array.from(options.values()), routeProtocolSet); } -function nativeToolOptionID(key: string, type: string): string { - return [key.trim(), type.trim()].filter(Boolean).join(":"); +function nativeToolOptionID(key: string, type: string, protocols: string[] = []): string { + return [ + key.trim(), + type.trim(), + ...protocols.map((protocol) => protocol.trim()).filter(Boolean).sort(), + ].filter(Boolean).join(":"); } function nativeToolMatchesRawTool(rawTool: Record, tool: NativeToolDefinition): boolean { @@ -662,8 +700,13 @@ export function normalizeModelCapabilitiesJSON( return Object.keys(payload).length > 0 ? JSON.stringify(payload, null, 2) : ""; } +function canonicalNativeToolProtocol(protocol: string): string { + const value = protocol.trim(); + return value.toLowerCase() === "google_generate_content" ? "gemini_generate_content" : value; +} + function formatNativeToolProtocols(protocols: string[]): string { - return protocols + return Array.from(new Set(protocols.map(canonicalNativeToolProtocol).filter(Boolean))) .map((protocol) => MODEL_OPTION_POLICY_PROTOCOL_LABELS[protocol as keyof typeof MODEL_OPTION_POLICY_PROTOCOL_LABELS] ?? protocol) .join(" / "); } @@ -673,14 +716,14 @@ function parseNativeToolProtocolsInput(value: string): string[] { new Set( value .split(",") - .map((item) => item.trim()) + .map(canonicalNativeToolProtocol) .filter(Boolean), ), ); } function formatNativeToolProtocolsInput(protocols: string[]): string { - return protocols.map((protocol) => protocol.trim()).filter(Boolean).join(", "); + return Array.from(new Set(protocols.map(canonicalNativeToolProtocol).filter(Boolean))).join(", "); } function nativeToolProtocolSelectOptions( @@ -817,7 +860,7 @@ function nativeToolRowFromConfig(value: Record, index: number): const type = typeof value.type === "string" ? value.type.trim() : nativeToolPayloadType(payload); const id = typeof value.id === "string" && value.id.trim() ? value.id.trim() - : nativeToolOptionID(key, type) || createCapabilityRowID(); + : nativeToolOptionID(key, type, protocols) || createCapabilityRowID(); if (!key && protocols.length === 0 && !type && Object.keys(payload).length === 0) { return null; } @@ -845,13 +888,46 @@ function parseNativeToolRows( const routeProtocolSet = new Set(routeProtocols.map((protocol) => resolveModelOptionPolicyProtocol(protocol)).filter(Boolean)); const rows = options.map((option) => nativeToolRowFromOption(option, false)); const applyRow = (row: NativeToolRow) => { - const id = nativeToolOptionID(row.key, row.type) || row.id; - const index = rows.findIndex((item) => item.id === id); - if (index < 0) { - rows.unshift({ ...row, id }); + const configuredProtocols = new Set( + parseNativeToolProtocolsInput(row.protocols) + .map((protocol) => resolveModelOptionPolicyProtocol(protocol)) + .filter(Boolean), + ); + const matchingIndexes = rows.flatMap((item, index) => { + if (item.key !== row.key || item.type !== row.type) { + return []; + } + const protocolMatched = configuredProtocols.size === 0 + || parseNativeToolProtocolsInput(item.protocols) + .some((protocol) => configuredProtocols.has(resolveModelOptionPolicyProtocol(protocol))); + return protocolMatched ? [index] : []; + }); + if (matchingIndexes.length === 0) { + rows.unshift({ ...row, id: row.id || createCapabilityRowID() }); return; } - rows[index] = { ...rows[index], ...row, id, catalog: rows[index].catalog }; + const configuredPayloadSignature = nativeToolPayloadSignature(JSON.parse(row.payload || "{}") as unknown); + for (const index of matchingIndexes) { + const catalogRow = rows[index]; + if (!catalogRow) { + continue; + } + const matchedProtocols = parseNativeToolProtocolsInput(catalogRow.protocols) + .filter((protocol) => configuredProtocols.size === 0 || configuredProtocols.has(resolveModelOptionPolicyProtocol(protocol))); + const catalogPayloadSignature = nativeToolPayloadSignature(JSON.parse(catalogRow.payload || "{}") as unknown); + rows[index] = { + ...catalogRow, + ...row, + id: catalogRow.id, + provider: row.provider || catalogRow.provider, + description: row.description || catalogRow.description, + protocols: formatNativeToolProtocolsInput(matchedProtocols), + payload: matchingIndexes.length === 1 || configuredPayloadSignature === catalogPayloadSignature + ? row.payload + : catalogRow.payload, + catalog: true, + }; + } }; if (Array.isArray(payload.nativeTools)) { diff --git a/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx b/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx index a12cf0f8c..20ddb8f62 100644 --- a/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx +++ b/frontend/features/admin/components/sections/models/models-capabilities-presets.tsx @@ -299,18 +299,20 @@ const MODEL_CAPABILITY_PRESETS: CapabilityPreset[] = [ nativeTools: [ { key: "google.code_execution", - protocols: ["google_generate_content", "gemini_generate_content"], + protocols: ["gemini_generate_content"], label: "Code Execution", enabled: true, defaultEnabled: true, payload: { code_execution: {}, }, + provider: "Google", type: "code_execution", + description: "Google hosted code execution tool.", }, { key: "google.google_search", - protocols: ["google_generate_content", "gemini_generate_content"], + protocols: ["gemini_generate_content"], label: "Google Search", enabled: true, defaultEnabled: true, @@ -323,14 +325,90 @@ const MODEL_CAPABILITY_PRESETS: CapabilityPreset[] = [ }, { key: "google.url_context", - protocols: ["google_generate_content", "gemini_generate_content"], + protocols: ["gemini_generate_content"], label: "URL Context", enabled: true, defaultEnabled: true, payload: { url_context: {}, }, + provider: "Google", + type: "url_context", + description: "Google hosted URL context tool.", + }, + ], + }, + }, + { + id: "gemini_interactions", + protocol: "gemini_interactions", + payload: { + defaultOptions: { + generation_config: { + thinking_level: "medium", + }, + }, + optionControls: [ + { + path: "generation_config.thinking_level", + type: "select", + label: "Thinking Level", + description: "Controls the depth of the model's internal reasoning.", + options: ["minimal", "low", "medium", "high"], + }, + { + path: "generation_config.thinking_summaries", + type: "select", + label: "Thinking Summaries", + description: "Controls whether thought summaries are included in the response.", + options: ["none", "auto"], + }, + { + path: "generation_config.max_output_tokens", + type: "number", + label: "Max Output Tokens", + description: "Maximum number of tokens to include in the response.", + }, + ], + nativeTools: [ + { + key: "google.code_execution", + protocols: ["gemini_interactions"], + label: "Code Execution", + enabled: true, + defaultEnabled: true, + payload: { + type: "code_execution", + }, + provider: "Google", + type: "code_execution", + description: "Google hosted code execution tool.", + }, + { + key: "google.google_search", + protocols: ["gemini_interactions"], + label: "Google Search", + enabled: true, + defaultEnabled: true, + payload: { + type: "google_search", + }, + provider: "Google", + type: "google_search", + description: "Google hosted search grounding tool.", + }, + { + key: "google.url_context", + protocols: ["gemini_interactions"], + label: "URL Context", + enabled: true, + defaultEnabled: true, + payload: { + type: "url_context", + }, + provider: "Google", type: "url_context", + description: "Google hosted URL context tool.", }, ], }, diff --git a/frontend/features/admin/model/conversation-settings.ts b/frontend/features/admin/model/conversation-settings.ts index 739002a74..88e65fb39 100644 --- a/frontend/features/admin/model/conversation-settings.ts +++ b/frontend/features/admin/model/conversation-settings.ts @@ -120,6 +120,21 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "size", "user" ], + "anthropic_messages": [ + "speed", + "top_k", + "cache_control", + "thinking.type", + "thinking.budget_tokens" + ], + "gemini_generate_content": [ + "generationConfig.temperature", + "generationConfig.topP", + "generationConfig.maxOutputTokens", + "generationConfig.responseMimeType", + "generationConfig.thinkingConfig.includeThoughts", + "generationConfig.thinkingConfig.thinkingLevel" + ], "google_image_generation": [ "generationConfig.responseModalities", "generationConfig.imageConfig.aspectRatio", @@ -130,6 +145,7 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "generation_config.top_p", "generation_config.max_output_tokens", "generation_config.thinking_level", + "generation_config.thinking_summaries", "response_format.type", "response_format.aspect_ratio", "response_format.image_size", @@ -141,13 +157,6 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "generationConfig.videoConfig.task", "generation_config.video_config.task" ], - "anthropic_messages": [ - "speed", - "top_k", - "cache_control", - "thinking.type", - "thinking.budget_tokens" - ], "xai_responses": [ "reasoning.effort", "min_p", @@ -171,12 +180,6 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "aspect_ratio", "duration", "resolution" - ], - "gemini_generate_content": [ - "generationConfig.temperature", - "generationConfig.topP", - "generationConfig.maxOutputTokens", - "generationConfig.responseMimeType" ] }`; diff --git a/frontend/features/chat/components/sections/chat-model-config.tsx b/frontend/features/chat/components/sections/chat-model-config.tsx index 9407f5d7c..dda5b4a41 100644 --- a/frontend/features/chat/components/sections/chat-model-config.tsx +++ b/frontend/features/chat/components/sections/chat-model-config.tsx @@ -115,6 +115,9 @@ const OPTION_LABEL_KEYS = new Set([ "generationConfig.thinkingConfig.includeThoughts", "generationConfig.thinkingConfig.thinkingBudget", "generationConfig.thinkingConfig.thinkingLevel", + "generation_config.max_output_tokens", + "generation_config.thinking_level", + "generation_config.thinking_summaries", "generationConfig.topK", "logprobs", "max_completion_tokens", diff --git a/frontend/features/chat/hooks/use-chat-model-options.ts b/frontend/features/chat/hooks/use-chat-model-options.ts index 39a23b518..59551340e 100644 --- a/frontend/features/chat/hooks/use-chat-model-options.ts +++ b/frontend/features/chat/hooks/use-chat-model-options.ts @@ -85,16 +85,16 @@ function normalizeNativeToolStrings(value: unknown): string[] { function nativeToolID({ key, - protocol, + protocols, type, index, }: { key: string; - protocol: string; + protocols: string[]; type: string; index: number; }): string { - return [key, protocol, type].map((item) => item.trim()).filter(Boolean).join(":") || `native-tool-${index}`; + return [key, ...protocols, type].map((item) => item.trim()).filter(Boolean).join(":") || `native-tool-${index}`; } function resolveNativeTools(raw: string): ModelNativeToolConfig[] { @@ -114,14 +114,15 @@ function resolveNativeTools(raw: string): ModelNativeToolConfig[] { const type = normalizeNativeToolString(source.type) || normalizeNativeToolString(payload.type); const protocol = normalizeNativeToolString(source.protocol); const protocols = normalizeNativeToolStrings(source.protocols); + const effectiveProtocols = protocols.length > 0 ? protocols : (protocol ? [protocol] : []); if (!key && !type && Object.keys(payload).length === 0) { return []; } return [{ - id: normalizeNativeToolString(source.id) || nativeToolID({ key, protocol, type, index }), + id: normalizeNativeToolString(source.id) || nativeToolID({ key, protocols: effectiveProtocols, type, index }), key, protocol, - protocols: protocols.length > 0 ? protocols : (protocol ? [protocol] : []), + protocols: effectiveProtocols, provider: normalizeNativeToolString(source.provider) || undefined, type, label: normalizeNativeToolString(source.label) || type || key, @@ -133,7 +134,7 @@ function resolveNativeTools(raw: string): ModelNativeToolConfig[] { }).filter((item) => item.enabled); } return resolveNativeToolKeys(raw).map((key, index) => ({ - id: nativeToolID({ key, protocol: "", type: "", index }), + id: nativeToolID({ key, protocols: [], type: "", index }), key, protocol: "", protocols: [], diff --git a/frontend/i18n/messages/en-US/admin-models.json b/frontend/i18n/messages/en-US/admin-models.json index b9149cd93..8eb7bde42 100644 --- a/frontend/i18n/messages/en-US/admin-models.json +++ b/frontend/i18n/messages/en-US/admin-models.json @@ -383,7 +383,7 @@ "toolsIntro": "Configure official native tools allowed for this model. Only tools enabled here can enter conversation settings.", "defaultsHelp": "Default parameters are written to defaultOptions. value is parsed as a JSON value, for example 0.7, true, null, \"high\", or an object.", "controlsHelp": "Parameter controls are written to optionControls. path maps to the real request option path, and final passthrough is still controlled by the option policy.", - "toolsHelp": "Official native tools are written to nativeTools. One tool can target multiple protocols; default-enabled tools are added to the user's default options automatically. User JSON tools are preserved by the backend only when they match these official tools.", + "toolsHelp": "Official native tools are written to nativeTools. Tools marked as default-enabled are added to the model's default request options; tools in user JSON are preserved only when they match an enabled official tool.", "addParameter": "Add", "addNativeTool": "Add", "emptyParameters": "No parameters", @@ -454,7 +454,7 @@ "policyTab": "Policy", "defaultsDescription": "defaultOptions becomes this model's default request parameters. The chat page uses it when the user has no saved local parameters for the model; backend option policy still applies before sending.", "controlsDescription": "optionControls only defines how the chat parameters dialog renders controls. It is not sent to the model by itself. path maps to the real options path.", - "toolsDescription": "nativeTools defines the provider-hosted official tools allowed for this model. Each tool is configured once, and protocols lists the applicable request protocols; a multi-protocol tool does not need duplicate rows.", + "toolsDescription": "nativeTools defines the provider-hosted official tools allowed for this model, and protocols specifies where each definition applies. Requests use only tool definitions that match the active protocol.", "toolsAutoDescription": "When users hand-write options.tools, only admin-allowed official tools are preserved. Parameters on allowed tools pass through; unknown tools and MCP/function tools cannot be injected through JSON.", "controlTypes": "type supports text, select, number, and boolean. select controls should provide options.", "policyDescription": "Rendering a control does not guarantee passthrough. The same path still needs to be allowed by the option policy; the backend filters options before sending." diff --git a/frontend/i18n/messages/en-US/chat.json b/frontend/i18n/messages/en-US/chat.json index b332728f6..545467ccf 100644 --- a/frontend/i18n/messages/en-US/chat.json +++ b/frontend/i18n/messages/en-US/chat.json @@ -689,6 +689,9 @@ "generationConfig__thinkingConfig__includeThoughts": "Include thoughts", "generationConfig__thinkingConfig__thinkingBudget": "Thinking budget", "generationConfig__thinkingConfig__thinkingLevel": "Thinking level", + "generation_config__max_output_tokens": "Max output tokens", + "generation_config__thinking_level": "Thinking level", + "generation_config__thinking_summaries": "Thinking summaries", "generationConfig__topK": "Top K", "logprobs": "Logprobs count", "max_completion_tokens": "Max output tokens", @@ -777,6 +780,9 @@ "generationConfig__thinkingConfig__includeThoughts": "Controls whether Gemini thinking content is included.", "generationConfig__thinkingConfig__thinkingBudget": "Gemini thinking token budget.", "generationConfig__thinkingConfig__thinkingLevel": "Gemini thinking level.", + "generation_config__max_output_tokens": "Limits the maximum number of tokens included in the Gemini response.", + "generation_config__thinking_level": "Controls the depth of thinking used by the Gemini model.", + "generation_config__thinking_summaries": "Controls whether the Gemini response includes thinking summaries.", "generationConfig__topK": "Gemini Top K sampling range.", "imageConfig__aspectRatio": "Google image output aspect ratio.", "imageConfig__imageSize": "Google image output size tier.", diff --git a/frontend/i18n/messages/zh-CN/admin-models.json b/frontend/i18n/messages/zh-CN/admin-models.json index 23d462600..a43eacee8 100644 --- a/frontend/i18n/messages/zh-CN/admin-models.json +++ b/frontend/i18n/messages/zh-CN/admin-models.json @@ -383,7 +383,7 @@ "toolsIntro": "配置该模型允许使用的官方原生工具；只有管理员在这里开启的官方工具才会进入会话配置。", "defaultsHelp": "默认参数会写入 defaultOptions。value 按 JSON 值解析，例如 0.7、true、null、\"high\" 或对象。", "controlsHelp": "参数控件会写入 optionControls。path 对应真实请求参数路径，最终是否透传仍由参数过滤规则决定。", - "toolsHelp": "原生官方工具会写入 nativeTools。同一个工具可配置多个适用协议；默认开启会自动进入用户默认参数；用户 JSON 中的 tools 只有命中这里的官方工具才会被后端保留。", + "toolsHelp": "官方原生工具会写入 nativeTools。设为默认开启的工具会加入模型默认请求参数；用户 JSON 中的 tools 仅在匹配已启用的官方工具时保留。", "addParameter": "新增", "addNativeTool": "新增", "emptyParameters": "暂无参数", @@ -454,7 +454,7 @@ "policyTab": "过滤规则", "defaultsDescription": "defaultOptions 会作为该模型的默认请求参数。用户未保存过本地参数时，聊天页会使用这里的值；最终发送前仍会经过后端参数策略。", "controlsDescription": "optionControls 只定义聊天参数 Dialog 如何展示控件，不会单独发送给模型。path 对应真实 options 路径。", - "toolsDescription": "nativeTools 定义当前模型允许的厂商官方原生工具。每个工具一行，protocols 表示适用协议列表；同一工具支持多个协议时不需要拆成多条配置。", + "toolsDescription": "nativeTools 定义当前模型允许使用的厂商官方原生工具，protocols 指定适用协议；请求时系统只会使用与当前协议匹配的工具定义。", "toolsAutoDescription": "用户侧手写 options.tools 时，只会保留管理员允许的官方工具；已允许工具的子参数会透传，未知工具和 MCP/function tool 不会通过 JSON 注入。", "controlTypes": "type 支持 text、select、number、boolean；select 需要提供 options。", "policyDescription": "控件展示不等于一定透传。对应 path 仍需要在参数过滤规则中允许，后端发送前会统一过滤。" diff --git a/frontend/i18n/messages/zh-CN/chat.json b/frontend/i18n/messages/zh-CN/chat.json index c03014bf9..7948330ce 100644 --- a/frontend/i18n/messages/zh-CN/chat.json +++ b/frontend/i18n/messages/zh-CN/chat.json @@ -689,6 +689,9 @@ "generationConfig__thinkingConfig__includeThoughts": "思考内容", "generationConfig__thinkingConfig__thinkingBudget": "思考预算", "generationConfig__thinkingConfig__thinkingLevel": "思考级别", + "generation_config__max_output_tokens": "最大输出 Tokens", + "generation_config__thinking_level": "思考级别", + "generation_config__thinking_summaries": "思考摘要", "generationConfig__topK": "Top K", "logprobs": "Logprobs 数量", "max_completion_tokens": "最大输出 Tokens", @@ -777,6 +780,9 @@ "generationConfig__thinkingConfig__includeThoughts": "控制是否包含 Gemini 思考内容。", "generationConfig__thinkingConfig__thinkingBudget": "Gemini 思考 token 预算。", "generationConfig__thinkingConfig__thinkingLevel": "Gemini 思考级别。", + "generation_config__max_output_tokens": "限制 Gemini 响应包含的最大 token 数。", + "generation_config__thinking_level": "控制 Gemini 模型的思考深度。", + "generation_config__thinking_summaries": "控制 Gemini 响应是否包含思考摘要。", "generationConfig__topK": "Gemini Top K 采样范围。", "imageConfig__aspectRatio": "Google 图片输出画幅比例。", "imageConfig__imageSize": "Google 图片输出尺寸档位。",