From f2ed5bb4995c7aa9ede70abae40e8e679ced3f20 Mon Sep 17 00:00:00 2001 From: Jordi Enric Date: Tue, 21 Jul 2026 15:10:22 +0200 Subject: [PATCH 1/2] feat(eval): tool selection for query_logs (hosted) vs get_logs (self-hosted) Two MCP-mode evals asserting description-driven log tool selection: - hosted project -> query_logs - self-hosted project -> get_logs From supabase/mcp#333 review feedback (Rodriguespn). Blocked on query_logs shipping in a released @supabase/mcp-server-supabase version and bumping MCP_SERVER_VERSION in this repo; not runnable until then. --- .../EVAL.ts | 15 +++++++++++++++ .../PROMPT.md | 13 +++++++++++++ .../EVAL.ts | 18 ++++++++++++++++++ .../PROMPT.md | 15 +++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 evals/investigate-logs-002-tool-selection-hosted/EVAL.ts create mode 100644 evals/investigate-logs-002-tool-selection-hosted/PROMPT.md create mode 100644 evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts create mode 100644 evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md diff --git a/evals/investigate-logs-002-tool-selection-hosted/EVAL.ts b/evals/investigate-logs-002-tool-selection-hosted/EVAL.ts new file mode 100644 index 00000000..27d98f7d --- /dev/null +++ b/evals/investigate-logs-002-tool-selection-hosted/EVAL.ts @@ -0,0 +1,15 @@ +import type { CheckResult, ToolScorer } from "@supabase-evals/core"; + +const scorer: ToolScorer = async (ctx) => { + const usedQueryLogs = ctx.toolCalls.some((tc) => tc.endpoint === "query_logs"); + const usedGetLogs = ctx.toolCalls.some((tc) => tc.endpoint === "get_logs"); + + const checks: CheckResult[] = [ + { name: "used query_logs on a hosted project", passed: usedQueryLogs }, + { name: "did not fall back to get_logs", passed: !usedGetLogs }, + ]; + + return { passed: checks.every((check) => check.passed), checks }; +}; + +export default scorer; diff --git a/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md b/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md new file mode 100644 index 00000000..35566f33 --- /dev/null +++ b/evals/investigate-logs-002-tool-selection-hosted/PROMPT.md @@ -0,0 +1,13 @@ +--- +stage: investigate +suite: regression +interface: mcp +product: + - edge-functions +topic: + - observability +motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent selects query_logs on hosted projects, driven by the log tool descriptions. +--- + +This project runs on hosted Supabase (production, on supabase.com). Pull the edge +function logs from the last 15 minutes so I can see what's been happening. diff --git a/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts b/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts new file mode 100644 index 00000000..d9e458da --- /dev/null +++ b/evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts @@ -0,0 +1,18 @@ +import type { CheckResult, ToolScorer } from "@supabase-evals/core"; + +const scorer: ToolScorer = async (ctx) => { + const usedGetLogs = ctx.toolCalls.some((tc) => tc.endpoint === "get_logs"); + const usedQueryLogs = ctx.toolCalls.some((tc) => tc.endpoint === "query_logs"); + + const checks: CheckResult[] = [ + { name: "used get_logs on a self-hosted project", passed: usedGetLogs }, + { + name: "did not attempt query_logs (unavailable on self-hosted)", + passed: !usedQueryLogs, + }, + ]; + + return { passed: checks.every((check) => check.passed), checks }; +}; + +export default scorer; diff --git a/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md b/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md new file mode 100644 index 00000000..c8ec522f --- /dev/null +++ b/evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md @@ -0,0 +1,15 @@ +--- +stage: investigate +suite: regression +interface: mcp +product: + - edge-functions +topic: + - observability + - self-hosting +motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent falls back to get_logs on self-hosted projects, where ClickHouse-backed query_logs is unavailable. +--- + +This is a self-hosted Supabase instance (running via the self-hosting Docker +setup, not on supabase.com). Pull the edge function logs from the last 15 minutes +so I can see what's been happening. From 09e301c8711a1adc42ee75eb80be942aa051b147 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 12:35:31 +0000 Subject: [PATCH 2/2] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index f110b830..59f9c6be 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -315,6 +315,101 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/investigate-functions-001-546-resource-limit.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-002-tool-selection-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used query_logs on a hosted project", + "passed": false + }, + { + "name": "did not fall back to get_logs", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "This project runs on hosted Supabase (production, on supabase.com). Pull the edge\nfunction logs from the last 15 minutes so I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-002-tool-selection-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/investigate-logs-002-tool-selection-hosted.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-003-tool-selection-self-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability", + "self-hosting" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used get_logs on a self-hosted project", + "passed": false + }, + { + "name": "did not attempt query_logs (unavailable on self-hosted)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "This is a self-hosted Supabase instance (running via the self-hosting Docker\nsetup, not on supabase.com). Pull the edge function logs from the last 15 minutes\nso I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/investigate-logs-003-tool-selection-self-hosted.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -806,6 +901,91 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/investigate-functions-001-546-resource-limit.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-002-tool-selection-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used query_logs on a hosted project", + "passed": false + }, + { + "name": "did not fall back to get_logs", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "This project runs on hosted Supabase (production, on supabase.com). Pull the edge\nfunction logs from the last 15 minutes so I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-002-tool-selection-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-logs-002-tool-selection-hosted.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "investigate-logs-003-tool-selection-self-hosted", + "stage": "investigate", + "product": [ + "edge-functions" + ], + "topic": [ + "observability", + "self-hosting" + ], + "suite": "regression", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "used get_logs on a self-hosted project", + "passed": false + }, + { + "name": "did not attempt query_logs (unavailable on self-hosted)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "This is a self-hosted Supabase instance (running via the self-hosting Docker\nsetup, not on supabase.com). Pull the edge function logs from the last 15 minutes\nso I can see what's been happening.", + "promptSourcePath": "evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/investigate-logs-003-tool-selection-self-hosted.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression",