From ed19736053575fe6ccbb42a23ee39f7f9bf5d826 Mon Sep 17 00:00:00 2001 From: duguwanglong Date: Mon, 20 Jul 2026 17:25:50 +0800 Subject: [PATCH] feat: make tool failure auto-disable configurable --- .flocks/flocks.json.example | 3 + flocks/config/config.py | 19 +++ flocks/server/routes/config.py | 58 +++++++++ flocks/tool/registry.py | 19 ++- tests/server/routes/test_remaining_routes.py | 37 ++++++ .../tool/test_failure_auto_disable_config.py | 97 +++++++++++++++ webui/src/api/toolFailureConfig.ts | 19 +++ webui/src/locales/en-US/nav.json | 7 +- webui/src/locales/zh-CN/nav.json | 7 +- webui/src/pages/Settings/index.test.tsx | 37 +++++- webui/src/pages/Settings/index.tsx | 113 ++++++++++++++++-- 11 files changed, 402 insertions(+), 14 deletions(-) create mode 100644 tests/tool/test_failure_auto_disable_config.py create mode 100644 webui/src/api/toolFailureConfig.ts diff --git a/.flocks/flocks.json.example b/.flocks/flocks.json.example index c60d4d838..1f8b273c7 100644 --- a/.flocks/flocks.json.example +++ b/.flocks/flocks.json.example @@ -39,6 +39,9 @@ "readMaxBytes": 51200, "readMaxLineLength": 2000 }, + "toolFailure": { + "disableOnRepeatedFailure": true + }, "sandbox": { "mode": "off", "scope": "agent", diff --git a/flocks/config/config.py b/flocks/config/config.py index 9e5fc41fe..65d151dba 100644 --- a/flocks/config/config.py +++ b/flocks/config/config.py @@ -349,6 +349,20 @@ class ToolOutputConfig(BaseModel): ) +class ToolFailureConfig(BaseModel): + """Repeated tool-failure handling.""" + + model_config = {"populate_by_name": True} + + disable_on_repeated_failure: bool = Field( + True, + alias="disableOnRepeatedFailure", + description=( + "Disable a standalone custom tool after repeated identical failures." + ), + ) + + class EnterpriseConfig(BaseModel): """Enterprise configuration""" @@ -675,6 +689,11 @@ class ConfigInfo(BaseModel): alias="toolOutput", description="Tool output size limits (read, truncation caps).", ) + tool_failure: Optional[ToolFailureConfig] = Field( + None, + alias="toolFailure", + description="Repeated tool-failure handling.", + ) experimental: Optional[ExperimentalConfig] = None # Memory system configuration (added for memory system integration) diff --git a/flocks/server/routes/config.py b/flocks/server/routes/config.py index 7f403f92c..6f706c281 100644 --- a/flocks/server/routes/config.py +++ b/flocks/server/routes/config.py @@ -141,6 +141,17 @@ class UIConfigUpdateRequest(BaseModel): display_name: Optional[str] = Field(None, alias="displayName") +class ToolFailurePreference(BaseModel): + """Repeated tool-failure preference exposed to the WebUI.""" + + model_config = {"populate_by_name": True} + + disable_on_repeated_failure: bool = Field( + ..., + alias="disableOnRepeatedFailure", + ) + + DEFAULT_UI_DISPLAY_NAME = "Flocks" DEFAULT_UI_PRO_DISPLAY_NAME = "Flocks Pro" FAVICON_MAX_BYTES = 512 * 1024 @@ -400,6 +411,12 @@ def _persist_ui_section(data: Dict[str, Any], ui_section: Dict[str, Any]) -> Non ConfigWriter._write_raw(data) +def _effective_tool_failure_preference(config: ConfigInfoModel) -> bool: + if config.tool_failure is None: + return True + return config.tool_failure.disable_on_repeated_failure + + @router.get("/ui-display", response_model=UIDisplayResponse, summary="Get public UI display name") async def get_ui_display() -> UIDisplayResponse: """Return only the effective WebUI display name for public screens.""" @@ -521,6 +538,47 @@ async def reset_ui_favicon() -> UIDisplayResponse: return await get_ui_display() +@router.get( + "/tool-failure", + response_model=ToolFailurePreference, + summary="Get repeated tool-failure preference", +) +async def get_tool_failure_preference() -> ToolFailurePreference: + """Return whether repeated identical failures automatically disable tools.""" + try: + config = await Config.get() + return ToolFailurePreference( + disableOnRepeatedFailure=_effective_tool_failure_preference(config) + ) + except Exception as e: + log.error("config.tool_failure.get.error", {"error": str(e)}) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.patch( + "/tool-failure", + response_model=ToolFailurePreference, + summary="Update repeated tool-failure preference", +) +async def update_tool_failure_preference( + request: ToolFailurePreference, +) -> ToolFailurePreference: + """Update only the repeated-failure switch in flocks.json.""" + try: + data = ConfigWriter._read_raw() + existing = data.get("toolFailure", data.get("tool_failure", {})) + section = dict(existing) if isinstance(existing, dict) else {} + section.pop("disable_on_repeated_failure", None) + section["disableOnRepeatedFailure"] = request.disable_on_repeated_failure + data.pop("tool_failure", None) + data["toolFailure"] = section + ConfigWriter._write_raw(data) + return await get_tool_failure_preference() + except Exception as e: + log.error("config.tool_failure.update.error", {"error": str(e)}) + raise HTTPException(status_code=400, detail=str(e)) + + @router.get("", summary="Get configuration") async def get_config() -> Dict[str, Any]: """ diff --git a/flocks/tool/registry.py b/flocks/tool/registry.py index d2bff0d7a..294450285 100644 --- a/flocks/tool/registry.py +++ b/flocks/tool/registry.py @@ -878,7 +878,11 @@ async def execute( if result.success: cls._reset_failure_state(tool_name) else: - disabled = cls._record_failure(tool, kwargs, result.error) + if await cls._failure_auto_disable_enabled(): + disabled = cls._record_failure(tool, kwargs, result.error) + else: + cls._reset_failure_state(tool_name) + disabled = False if disabled: result.metadata = {**(result.metadata or {}), "disabled": True, "disabled_reason": "repeated_error"} suffix = f"tool disabled after {cls._failure_disable_threshold} identical errors" @@ -1743,6 +1747,19 @@ def _reset_failure_state(cls, tool_name: str) -> None: if tool_name in cls._failure_state: cls._failure_state.pop(tool_name, None) + @classmethod + async def _failure_auto_disable_enabled(cls) -> bool: + """Return the configured repeated-failure behavior, defaulting on.""" + try: + from flocks.config.config import Config + + config = await Config.get() + if config.tool_failure is not None: + return config.tool_failure.disable_on_repeated_failure + except Exception: + pass + return True + @classmethod def _should_track_failure(cls, tool: Tool) -> bool: """Track failures only for standalone custom tools. diff --git a/tests/server/routes/test_remaining_routes.py b/tests/server/routes/test_remaining_routes.py index ce681b707..ebb703852 100644 --- a/tests/server/routes/test_remaining_routes.py +++ b/tests/server/routes/test_remaining_routes.py @@ -508,6 +508,43 @@ async def test_config_has_expected_top_level_keys(self, client: AsyncClient): f"No expected keys found. Got: {list(data.keys())}" ) + @pytest.mark.asyncio + async def test_tool_failure_preference_defaults_on_and_updates_only_its_section( + self, + client: AsyncClient, + tmp_path, + monkeypatch, + ): + from flocks.config.config import Config + from flocks.config.config_writer import ConfigWriter + + monkeypatch.setenv("FLOCKS_CONFIG_DIR", str(tmp_path / "config")) + Config._global_config = None + Config._cached_config = None + ConfigWriter._write_raw({"theme": "dark"}) + + resp = await client.get("/api/config/tool-failure") + assert resp.status_code == status.HTTP_200_OK + assert resp.json() == {"disableOnRepeatedFailure": True} + + resp = await client.patch( + "/api/config/tool-failure", + json={"disableOnRepeatedFailure": False}, + ) + assert resp.status_code == status.HTTP_200_OK + assert resp.json() == {"disableOnRepeatedFailure": False} + assert ConfigWriter._read_raw() == { + "theme": "dark", + "toolFailure": {"disableOnRepeatedFailure": False}, + } + + resp = await client.get("/api/config/tool-failure") + assert resp.status_code == status.HTTP_200_OK + assert resp.json() == {"disableOnRepeatedFailure": False} + + resp = await client.patch("/api/config/tool-failure", json={}) + assert resp.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + @pytest.mark.asyncio async def test_ui_display_defaults_and_updates( self, diff --git a/tests/tool/test_failure_auto_disable_config.py b/tests/tool/test_failure_auto_disable_config.py new file mode 100644 index 000000000..2f3bf692b --- /dev/null +++ b/tests/tool/test_failure_auto_disable_config.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import pytest + +from flocks.config.config import Config, ConfigInfo, ToolFailureConfig +from flocks.tool.registry import ( + Tool, + ToolCategory, + ToolContext, + ToolInfo, + ToolRegistry, + ToolResult, +) + + +def _failing_tool(name: str = "failing_custom_tool") -> Tool: + async def handler(_ctx: ToolContext, **_kwargs) -> ToolResult: + return ToolResult(success=False, error="synthetic repeated failure") + + return Tool( + info=ToolInfo( + name=name, + description="Always fails for repeated-failure tests", + category=ToolCategory.CUSTOM, + source="plugin_py", + ), + handler=handler, + ) + + +@pytest.fixture +def isolated_failure_tracking(monkeypatch: pytest.MonkeyPatch): + tool = _failing_tool() + monkeypatch.setattr(ToolRegistry, "_initialized", True) + monkeypatch.setattr(ToolRegistry, "_tools", {tool.info.name: tool}) + monkeypatch.setattr(ToolRegistry, "_failure_state", {}) + monkeypatch.setattr(Config, "_cached_config", ConfigInfo()) + return tool + + +def test_tool_failure_config_defaults_to_enabled() -> None: + section = ToolFailureConfig() + + assert section.disable_on_repeated_failure is True + + +def test_tool_failure_config_accepts_camel_and_snake_case() -> None: + camel = ConfigInfo.model_validate( + {"toolFailure": {"disableOnRepeatedFailure": False}} + ) + snake = ConfigInfo.model_validate( + {"tool_failure": {"disable_on_repeated_failure": False}} + ) + + assert camel.tool_failure is not None + assert camel.tool_failure.disable_on_repeated_failure is False + assert snake.tool_failure is not None + assert snake.tool_failure.disable_on_repeated_failure is False + + +@pytest.mark.asyncio +async def test_repeated_failures_disable_tool_by_default( + isolated_failure_tracking: Tool, +) -> None: + tool = isolated_failure_tracking + + for _ in range(ToolRegistry._failure_disable_threshold): + result = await ToolRegistry.execute(tool.info.name, query="same") + + assert tool.info.enabled is False + assert result.metadata == { + "disabled": True, + "disabled_reason": "repeated_error", + } + + +@pytest.mark.asyncio +async def test_config_can_turn_off_repeated_failure_auto_disable( + isolated_failure_tracking: Tool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + tool = isolated_failure_tracking + monkeypatch.setattr( + Config, + "_cached_config", + ConfigInfo.model_validate( + {"toolFailure": {"disableOnRepeatedFailure": False}} + ), + ) + + for _ in range(ToolRegistry._failure_disable_threshold + 1): + result = await ToolRegistry.execute(tool.info.name, query="same") + + assert result.success is False + assert "disabled" not in result.metadata + assert tool.info.enabled is True + assert ToolRegistry._failure_state == {} diff --git a/webui/src/api/toolFailureConfig.ts b/webui/src/api/toolFailureConfig.ts new file mode 100644 index 000000000..e45333cba --- /dev/null +++ b/webui/src/api/toolFailureConfig.ts @@ -0,0 +1,19 @@ +import client from './client'; + +export interface ToolFailureConfig { + disableOnRepeatedFailure: boolean; +} + +export const toolFailureConfigApi = { + async get(): Promise { + const response = await client.get('/api/config/tool-failure'); + return response.data; + }, + + async update(disableOnRepeatedFailure: boolean): Promise { + const response = await client.patch('/api/config/tool-failure', { + disableOnRepeatedFailure, + }); + return response.data; + }, +}; diff --git a/webui/src/locales/en-US/nav.json b/webui/src/locales/en-US/nav.json index e66d43e24..72e713ef5 100644 --- a/webui/src/locales/en-US/nav.json +++ b/webui/src/locales/en-US/nav.json @@ -27,7 +27,7 @@ "settingsTitle": "Settings", "settingsDescription": "Manage preferences, account, and system settings", "settingsPreferences": "Preferences", - "settingsPreferencesDescription": "Adjust the workspace name, console language, and interface theme.", + "settingsPreferencesDescription": "Adjust workspace display, language, theme, and tool failure handling.", "settingsGroupPreferences": "Preferences", "settingsGroupSystem": "Account & System", "settingsGroupIntegrations": "Models & Channels", @@ -53,6 +53,11 @@ "themeDescription": "Choose the default light or dark mode for the console.", "lightTheme": "Light", "darkTheme": "Dark", + "toolFailureAutoDisable": "Auto-disable repeatedly failing tools", + "toolFailureAutoDisableDescription": "Disable a standalone custom tool after the same parameters repeatedly produce the same error up to the configured threshold.", + "toolFailureSettingSaved": "Tool failure handling preference saved", + "toolFailureSettingSaveFailed": "Failed to save tool failure handling preference", + "toolFailureSettingLoadFailed": "Failed to load tool failure handling preference", "logout": "Log out", "expandNav": "Expand navigation", "collapseNav": "Collapse navigation", diff --git a/webui/src/locales/zh-CN/nav.json b/webui/src/locales/zh-CN/nav.json index abed9b104..752a117b6 100644 --- a/webui/src/locales/zh-CN/nav.json +++ b/webui/src/locales/zh-CN/nav.json @@ -27,7 +27,7 @@ "settingsTitle": "设置", "settingsDescription": "管理偏好、账号与系统配置", "settingsPreferences": "偏好设置", - "settingsPreferencesDescription": "调整当前工作台的名称、显示语言和界面主题。", + "settingsPreferencesDescription": "调整当前工作台的显示、语言、主题和工具失败处理。", "settingsGroupPreferences": "偏好", "settingsGroupSystem": "账号与系统", "settingsGroupIntegrations": "模型与通道", @@ -53,6 +53,11 @@ "themeDescription": "选择控制台默认使用的浅色或深色模式。", "lightTheme": "浅色", "darkTheme": "深色", + "toolFailureAutoDisable": "自动禁用重复失败工具", + "toolFailureAutoDisableDescription": "同一独立自定义工具以相同参数重复返回相同错误达到阈值后，自动禁用该工具。", + "toolFailureSettingSaved": "工具失败处理设置已保存", + "toolFailureSettingSaveFailed": "保存工具失败处理设置失败", + "toolFailureSettingLoadFailed": "加载工具失败处理设置失败", "logout": "退出登录", "expandNav": "展开导航", "collapseNav": "收起导航", diff --git a/webui/src/pages/Settings/index.test.tsx b/webui/src/pages/Settings/index.test.tsx index 35d5c605d..19a266cd1 100644 --- a/webui/src/pages/Settings/index.test.tsx +++ b/webui/src/pages/Settings/index.test.tsx @@ -1,17 +1,21 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { render, screen, within } from '@testing-library/react'; +import { render, screen, waitFor, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { MemoryRouter, Route, Routes, useLocation } from 'react-router-dom'; import SettingsPage from './index'; import { ThemeContext, type Theme } from '@/contexts/ThemeContext'; import { ToastProvider } from '@/components/common/Toast'; -const { changeLanguage, flocksproUsersApi, setTheme, useAuth } = vi.hoisted(() => ({ +const { changeLanguage, flocksproUsersApi, setTheme, toolFailureConfigApi, useAuth } = vi.hoisted(() => ({ changeLanguage: vi.fn(), flocksproUsersApi: { hasCapability: vi.fn(), }, setTheme: vi.fn(), + toolFailureConfigApi: { + get: vi.fn(), + update: vi.fn(), + }, useAuth: vi.fn(), })); @@ -33,6 +37,10 @@ vi.mock('@/api/flocksproUsers', () => ({ flocksproUsersApi, })); +vi.mock('@/api/toolFailureConfig', () => ({ + toolFailureConfigApi, +})); + vi.mock('@/pages/Config', () => ({ default: () =>
account page
, })); @@ -92,6 +100,10 @@ describe('SettingsPage', () => { beforeEach(() => { vi.clearAllMocks(); flocksproUsersApi.hasCapability.mockResolvedValue(true); + toolFailureConfigApi.get.mockResolvedValue({ disableOnRepeatedFailure: true }); + toolFailureConfigApi.update.mockImplementation(async (disableOnRepeatedFailure: boolean) => ({ + disableOnRepeatedFailure, + })); useAuth.mockReturnValue({ user: { id: 'user-1', @@ -117,6 +129,27 @@ describe('SettingsPage', () => { expect(setTheme).toHaveBeenCalledWith('dark'); }); + it('loads and updates repeated tool failure auto-disable', async () => { + const user = userEvent.setup(); + toolFailureConfigApi.get.mockResolvedValue({ disableOnRepeatedFailure: false }); + + renderSettings('/settings/preferences'); + + const autoDisableSwitch = await screen.findByRole('switch', { + name: 'toolFailureAutoDisable', + }); + await waitFor(() => { + expect(autoDisableSwitch).toHaveAttribute('aria-checked', 'false'); + }); + + await user.click(autoDisableSwitch); + + await waitFor(() => { + expect(toolFailureConfigApi.update).toHaveBeenCalledWith(true); + expect(autoDisableSwitch).toHaveAttribute('aria-checked', 'true'); + }); + }); + it('redirects legacy model and channel settings URLs to workspace pages', async () => { const { unmount } = renderSettings('/settings/models'); diff --git a/webui/src/pages/Settings/index.tsx b/webui/src/pages/Settings/index.tsx index 337cd9e92..ba0c8238b 100644 --- a/webui/src/pages/Settings/index.tsx +++ b/webui/src/pages/Settings/index.tsx @@ -26,6 +26,7 @@ import { useAuth } from '@/contexts/AuthContext'; import { useProductName } from '@/contexts/ProductNameContext'; import { useToast } from '@/components/common/Toast'; import { flocksproUsersApi } from '@/api/flocksproUsers'; +import { toolFailureConfigApi } from '@/api/toolFailureConfig'; import { preloadI18nNamespaces } from '@/i18nResources'; type LazySettingsModule = { default: ComponentType }; @@ -156,6 +157,38 @@ function SegmentedOption({ ); } +function PreferenceSwitch({ + checked, + disabled, + label, + onChange, +}: { + checked: boolean; + disabled: boolean; + label: string; + onChange: () => void; +}) { + return ( + + ); +} + function PreferencesPanel() { const { t, i18n } = useTranslation('nav'); const { theme, setTheme } = useContext(ThemeContext); @@ -168,26 +201,58 @@ function PreferencesPanel() { uploadProductFavicon, resetProductFavicon, } = useProductName(); - const toast = useToast(); + const { error: showToastError, success: showToastSuccess } = useToast(); const language = i18n.language?.toLowerCase().startsWith('zh') ? 'zh-CN' : 'en-US'; const faviconInputRef = useRef(null); const [displayNameDraft, setDisplayNameDraft] = useState(configuredDisplayName ?? ''); const [savingDisplayName, setSavingDisplayName] = useState(false); const [savingFavicon, setSavingFavicon] = useState(false); + const [toolFailureAutoDisable, setToolFailureAutoDisable] = useState(true); + const [loadingToolFailure, setLoadingToolFailure] = useState(true); + const [savingToolFailure, setSavingToolFailure] = useState(false); const normalizedDisplayName = displayNameDraft.trim(); const displayNameChanged = normalizedDisplayName !== (configuredDisplayName ?? ''); + const toolFailureSettingLoadFailedMessage = t('toolFailureSettingLoadFailed'); useEffect(() => { setDisplayNameDraft(configuredDisplayName ?? ''); }, [configuredDisplayName]); + useEffect(() => { + let cancelled = false; + setLoadingToolFailure(true); + toolFailureConfigApi.get() + .then((config) => { + if (!cancelled) { + setToolFailureAutoDisable(config.disableOnRepeatedFailure); + } + }) + .catch((err: any) => { + if (!cancelled) { + showToastError( + toolFailureSettingLoadFailedMessage, + err?.response?.data?.detail || err?.message, + ); + } + }) + .finally(() => { + if (!cancelled) { + setLoadingToolFailure(false); + } + }); + + return () => { + cancelled = true; + }; + }, [showToastError, toolFailureSettingLoadFailedMessage]); + const handleSaveDisplayName = async () => { setSavingDisplayName(true); try { await updateProductName(normalizedDisplayName || null); - toast.success(t('displayNameSaved')); + showToastSuccess(t('displayNameSaved')); } catch (err: any) { - toast.error(t('displayNameSaveFailed'), err?.response?.data?.detail || err?.message); + showToastError(t('displayNameSaveFailed'), err?.response?.data?.detail || err?.message); } finally { setSavingDisplayName(false); } @@ -197,9 +262,9 @@ function PreferencesPanel() { setSavingDisplayName(true); try { await updateProductName(null); - toast.success(t('displayNameSaved')); + showToastSuccess(t('displayNameSaved')); } catch (err: any) { - toast.error(t('displayNameSaveFailed'), err?.response?.data?.detail || err?.message); + showToastError(t('displayNameSaveFailed'), err?.response?.data?.detail || err?.message); } finally { setSavingDisplayName(false); } @@ -213,9 +278,9 @@ function PreferencesPanel() { setSavingFavicon(true); try { await uploadProductFavicon(file); - toast.success(t('faviconSaved')); + showToastSuccess(t('faviconSaved')); } catch (err: any) { - toast.error(t('faviconSaveFailed'), err?.response?.data?.detail || err?.message); + showToastError(t('faviconSaveFailed'), err?.response?.data?.detail || err?.message); } finally { setSavingFavicon(false); } @@ -225,14 +290,31 @@ function PreferencesPanel() { setSavingFavicon(true); try { await resetProductFavicon(); - toast.success(t('faviconSaved')); + showToastSuccess(t('faviconSaved')); } catch (err: any) { - toast.error(t('faviconSaveFailed'), err?.response?.data?.detail || err?.message); + showToastError(t('faviconSaveFailed'), err?.response?.data?.detail || err?.message); } finally { setSavingFavicon(false); } }; + const handleToolFailureAutoDisableChange = async () => { + const nextValue = !toolFailureAutoDisable; + setSavingToolFailure(true); + try { + const config = await toolFailureConfigApi.update(nextValue); + setToolFailureAutoDisable(config.disableOnRepeatedFailure); + showToastSuccess(t('toolFailureSettingSaved')); + } catch (err: any) { + showToastError( + t('toolFailureSettingSaveFailed'), + err?.response?.data?.detail || err?.message, + ); + } finally { + setSavingToolFailure(false); + } + }; + return (
@@ -371,6 +453,19 @@ function PreferencesPanel() {
+ + + void handleToolFailureAutoDisableChange()} + /> + );