feat(eval): log tool selection — query_logs (hosted) vs get_logs (self-hosted)

apps/web/src/data/regression-eval-results.json

@@ -315,6 +315,101 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5/investigate-functions-001-546-resource-limit.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-logs-002-tool-selection-hosted",
+    "stage": "investigate",
+    "product": [
+      "edge-functions"
+    ],
+    "topic": [
+      "observability"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "used query_logs on a hosted project",
+        "passed": false
+      },
+      {
+        "name": "did not fall back to get_logs",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "This project runs on hosted Supabase (production, on supabase.com). Pull the edge\nfunction logs from the last 15 minutes so I can see what's been happening.",
+    "promptSourcePath": "evals/investigate-logs-002-tool-selection-hosted/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5/investigate-logs-002-tool-selection-hosted.json"
+  },
+  {
+    "experiment": "claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-logs-003-tool-selection-self-hosted",
+    "stage": "investigate",
+    "product": [
+      "edge-functions"
+    ],
+    "topic": [
+      "observability",
+      "self-hosting"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "used get_logs on a self-hosted project",
+        "passed": false
+      },
+      {
+        "name": "did not attempt query_logs (unavailable on self-hosted)",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "This is a self-hosted Supabase instance (running via the self-hosting Docker\nsetup, not on supabase.com). Pull the edge function logs from the last 15 minutes\nso I can see what's been happening.",
+    "promptSourcePath": "evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5/investigate-logs-003-tool-selection-self-hosted.json"
+  },
   {
     "experiment": "claude-code-sonnet-5",
     "experimentSuite": "regression",
@@ -806,6 +901,91 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5-no-skills/investigate-functions-001-546-resource-limit.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5-no-skills",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-logs-002-tool-selection-hosted",
+    "stage": "investigate",
+    "product": [
+      "edge-functions"
+    ],
+    "topic": [
+      "observability"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "used query_logs on a hosted project",
+        "passed": false
+      },
+      {
+        "name": "did not fall back to get_logs",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "This project runs on hosted Supabase (production, on supabase.com). Pull the edge\nfunction logs from the last 15 minutes so I can see what's been happening.",
+    "promptSourcePath": "evals/investigate-logs-002-tool-selection-hosted/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5-no-skills/investigate-logs-002-tool-selection-hosted.json"
+  },
+  {
+    "experiment": "claude-code-sonnet-5-no-skills",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-logs-003-tool-selection-self-hosted",
+    "stage": "investigate",
+    "product": [
+      "edge-functions"
+    ],
+    "topic": [
+      "observability",
+      "self-hosting"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "used get_logs on a self-hosted project",
+        "passed": false
+      },
+      {
+        "name": "did not attempt query_logs (unavailable on self-hosted)",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "This is a self-hosted Supabase instance (running via the self-hosting Docker\nsetup, not on supabase.com). Pull the edge function logs from the last 15 minutes\nso I can see what's been happening.",
+    "promptSourcePath": "evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5-no-skills/investigate-logs-003-tool-selection-self-hosted.json"
+  },
   {
     "experiment": "claude-code-sonnet-5-no-skills",
     "experimentSuite": "regression",

evals/investigate-logs-002-tool-selection-hosted/EVAL.ts

@@ -0,0 +1,15 @@
+import type { CheckResult, ToolScorer } from "@supabase-evals/core";
+
+const scorer: ToolScorer = async (ctx) => {
+  const usedQueryLogs = ctx.toolCalls.some((tc) => tc.endpoint === "query_logs");
+  const usedGetLogs = ctx.toolCalls.some((tc) => tc.endpoint === "get_logs");
+
+  const checks: CheckResult[] = [
+    { name: "used query_logs on a hosted project", passed: usedQueryLogs },
+    { name: "did not fall back to get_logs", passed: !usedGetLogs },
+  ];
+
+  return { passed: checks.every((check) => check.passed), checks };
+};
+
+export default scorer;

evals/investigate-logs-002-tool-selection-hosted/PROMPT.md

@@ -0,0 +1,13 @@
+---
+stage: investigate
+suite: regression
+interface: mcp
+product:
+  - edge-functions
+topic:
+  - observability
+motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent selects query_logs on hosted projects, driven by the log tool descriptions.
+---
+
+This project runs on hosted Supabase (production, on supabase.com). Pull the edge
+function logs from the last 15 minutes so I can see what's been happening.

evals/investigate-logs-003-tool-selection-self-hosted/EVAL.ts

@@ -0,0 +1,18 @@
+import type { CheckResult, ToolScorer } from "@supabase-evals/core";
+
+const scorer: ToolScorer = async (ctx) => {
+  const usedGetLogs = ctx.toolCalls.some((tc) => tc.endpoint === "get_logs");
+  const usedQueryLogs = ctx.toolCalls.some((tc) => tc.endpoint === "query_logs");
+
+  const checks: CheckResult[] = [
+    { name: "used get_logs on a self-hosted project", passed: usedGetLogs },
+    {
+      name: "did not attempt query_logs (unavailable on self-hosted)",
+      passed: !usedQueryLogs,
+    },
+  ];
+
+  return { passed: checks.every((check) => check.passed), checks };
+};
+
+export default scorer;

evals/investigate-logs-003-tool-selection-self-hosted/PROMPT.md

@@ -0,0 +1,15 @@
+---
+stage: investigate
+suite: regression
+interface: mcp
+product:
+  - edge-functions
+topic:
+  - observability
+  - self-hosting
+motivation: supabase/mcp#333 review (Rodriguespn) — verify the agent falls back to get_logs on self-hosted projects, where ClickHouse-backed query_logs is unavailable.
+---
+
+This is a self-hosted Supabase instance (running via the self-hosting Docker
+setup, not on supabase.com). Pull the edge function logs from the last 15 minutes
+so I can see what's been happening.