From 4460165b522d4c6bfbe8c277440487c986c453f5 Mon Sep 17 00:00:00 2001 From: mattrossman <22670878+mattrossman@users.noreply.github.com> Date: Thu, 6 Aug 2026 07:17:59 +0000 Subject: [PATCH] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 351 +++++++++--------- 1 file changed, 169 insertions(+), 182 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 97c715da..db89acba 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -32,7 +32,7 @@ { "name": "suggested relevant next steps for a Supabase + Next.js project", "passed": true, - "judgeNotes": "The response gives several concrete Supabase/Next.js next steps: running `supabase start`, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` with API URL and anon key, creating migrations, and linking to a hosted project." + "judgeNotes": "The response includes concrete Supabase/Next.js next steps: running `supabase start`, linking a remote project, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `NEXT_PUBLIC_SUPABASE_URL` and anon key env vars, and creating a first migration." }, { "name": "attempted `npx plugins add supabase-community/supabase-plugin`", @@ -90,17 +90,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"78261957-f24f-402c-9b0c-6c34789a6142\",\"metric\":\"steps_a_msfr686p\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"770dfc19-d8cf-4a9d-b5ef-7e1bb36591fe\",\"metric\":\"steps_a_msh6nql3\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"78261957-f24f-402c-9b0c-6c34789a6142\",\"metric\":\"steps_a_msfr686p\",\"value\":111}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"770dfc19-d8cf-4a9d-b5ef-7e1bb36591fe\",\"metric\":\"steps_a_msh6nql3\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"92a37897-ec79-465b-8979-502e201d2851\",\"metric\":\"steps_b_msfr686p\",\"value\":222}]}" + "notes": "status 200: {\"data\":[{\"user_id\":\"3df8692c-3832-4c68-9b20-5ecfd0cec116\",\"metric\":\"steps_b_msh6nql3\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -133,84 +133,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server npm package edge functions withSupabase multi-auth\", limit: 10) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/recursive-functions", - "title": "Recursive / Nested Function Calls" - }, - { - "url": "https://supabase.com/docs/guides/functions/cors", - "title": "CORS (Cross-Origin Resource Sharing) support for Invoking from the browser" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", - "title": "Resumable WebSockets with Edge Functions" - } - ], - "resultChars": 111253 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"verify_jwt combining auth modes secret key apikey header config.toml functions\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/deploy", - "title": "Deploy to Production" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-tips", - "title": "Development tips" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-headers", - "title": "Authorization headers" - }, - { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 28146 - } - ] + "calls": [] }, "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nImplement it with the **`@supabase/server`** package, which is built for exactly\nthis kind of multi-auth Edge Function. Import it directly in your function:\n\n```ts\nimport { withSupabase } from \"npm:@supabase/server\";\n```\n\nOur product stores per-user metrics in a `user_stats` table that already exists\n(see `supabase/migrations/`), protected by row-level security so a user can read\nonly their own rows.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.\n\nGet the local stack running so the function is reachable at the path above.", "promptSourcePath": "evals/build-functions-006-dual-auth-with-server/PROMPT.md", @@ -246,7 +169,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treated the issue as a Supabase Realtime/Postgres Changes setup, enabled the `public.messages` table in the `supabase_realtime` publication, verified it, and provided `postgres_changes` client subscription code. It did not recommend or imply read replicas are needed or useful for live updates." + "judgeNotes": "The assistant correctly treated the issue as Supabase Realtime/Postgres Changes setup, inspected and added the messages table to the supabase_realtime publication, and provided postgres_changes client subscription code. It did not recommend read replicas or confuse them with Realtime/logical replication." } ], "skills": { @@ -262,31 +185,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"enable realtime postgres changes subscribe table broadcast\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"enable realtime postgres changes publication supabase_realtime add table\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, { "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", "title": "Subscribing to Database Changes" }, { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" }, { "url": "https://supabase.com/docs/guides/realtime/benchmarks", "title": "Benchmarks" }, { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" } ], - "resultChars": 150974 + "resultChars": 115589 } ] }, @@ -318,17 +241,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "The assistant correctly identified `video-thumbnails` as the affected function and explicitly stated the failing calls are HTTP 546 / WORKER_RESOURCE_LIMIT due to CPUTime resource limits, not 500s or 503s." + "judgeNotes": "Identified video-thumbnails as affected and correctly described failures as HTTP 546 due to CPU/resource limit, not 500/503." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "Assistant specifically attributes the 546 failures to CPU time exhaustion, citing shutdown reason CPUTime and cpu_time_used=2000ms at the CPU limit." + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason `CPUTime` and `cpu_time_used` pinned at `cpu_time_limit` 2000ms. It also distinguishes this from wall-clock time and code exceptions." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant clearly identifies CPUTime resource limit and recommends moving thumbnail extraction out of the Edge Function to a background worker/external service, plus reducing CPU via downscaling/pre-trimming. This directly addresses reducing/offloading CPU-intensive work." + "judgeNotes": "The assistant clearly identifies CPUTime limit from CPU-bound thumbnail generation and recommends architectural fixes that reduce/offload CPU work: moving transcoding/thumbnail generation out of the Edge Function to a queue/background worker or managed media API, and reducing work per invocation as a short-term mitigation. It notes the limit cannot be raised and retry is only a band-aid." } ], "skills": { @@ -344,31 +267,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"edge function CPU time limit shutdown reason\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"edge function CPU time limit shutdown\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", - "title": "Edge Function shutdown reasons explained" - }, { "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", "title": "Understanding Edge Function CPU limits" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-functions-worker-timeouts-and-websocket-drops", - "title": "Edge Functions worker timeouts and WebSocket drops" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", + "title": "Edge Function 'wall clock time limit reached'" }, { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", + "title": "Edge Function shutdown reasons explained" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-monitoring-resource-usage", - "title": "Monitoring Edge Function resource usage" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-functions-worker-timeouts-and-websocket-drops", + "title": "Edge Functions worker timeouts and WebSocket drops" + }, + { + "url": "https://supabase.com/docs/guides/functions/limits", + "title": "Limits" } ], - "resultChars": 25339 + "resultChars": 24082 } ] }, @@ -444,7 +367,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnoses missing Data API/table grants separately from RLS, preserves owner-scoped RLS, grants only SELECT and INSERT on public.journal_entries to authenticated, and does not grant anon/public or weaken policies. It does not explicitly mention api.auto_expose_new_tables disabled, but it effectively recognizes secure-by-default explicit grants as the root cause." + "judgeNotes": "Diagnosed missing table-level grants as distinct from RLS, kept owner-scoped RLS intact, granted only SELECT/INSERT on public.journal_entries to authenticated, and did not grant anon/public or disable/weaken RLS. It also recognized secure-by-default restricted exposure/default ACL behavior for SQL-created tables." } ], "skills": { @@ -492,7 +415,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"df7e215e-6690-4608-8c86-72312390b5ab\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"c06ce659-2c6a-47fc-b001-4e113c75d201\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -501,7 +424,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "Diagnosed the zero-row UPDATE as the tasks UPDATE RLS policy missing a USING clause, explained WITH CHECK alone is insufficient for finding existing rows, and fixed it with an authenticated-role policy using the task owner user_id while retaining WITH CHECK to prevent reassignment. RLS was kept enabled and no blanket/public policy was used." + "judgeNotes": "Diagnosed missing USING clause on the UPDATE RLS policy, explained silent 0-row behavior, and fixed with an authenticated owner-scoped USING plus retained WITH CHECK. RLS remains enabled and policy is not overly broad." } ], "skills": { @@ -514,36 +437,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"RLS UPDATE policy requires SELECT policy USING WITH CHECK\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", - "title": "RLS Simplified" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - } - ], - "resultChars": 79348 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", @@ -573,7 +467,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer clearly says restart is the better first step and that pause/restore is not appropriate for transient unhealthy recovery. It distinguishes restart from pause/restore/backup-style recovery, gives concrete next steps including checking advisors/logs and scaling/resource investigation, and avoids recommending pause plus restore as the recovery path." + "judgeNotes": "Answer clearly says restart, not pause/restore; distinguishes pause/restore from restart; provides concrete first steps including checking reports/logs/advisors and investigating resource issues. It could mention contacting support and scaling/reducing workload more explicitly, but overall meets safe docs-grounded recovery guidance." } ], "skills": { @@ -589,31 +483,87 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"project unhealthy dashboard restart pause restore troubleshooting\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project unhealthy restart pause restore troubleshooting\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", - "title": "Project Status reports unhealthy services" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" }, { - "url": "https://supabase.com/docs/guides/platform/free-project-pausing", - "title": "Project Pausing" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/mcp-authentication", + "title": "Model Context Protocol (MCP) Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", + "title": "Configure Phone Login & MFA" + } + ], + "resultChars": 68009 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restart project vs pause and restore dashboard unresponsive\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { "url": "https://supabase.com/docs/guides/platform/backups", "title": "Database Backups" }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + }, { "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", "title": "Restoring a downloaded backup locally" + }, + { + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" + } + ], + "resultChars": 44530 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restart project database high CPU disk full unresponsive troubleshooting\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/disk-size-not-shrinking-after-deleting-data-135390", + "title": "Disk size not shrinking after deleting data" + }, + { + "url": "https://supabase.com/docs/guides/database/prisma/prisma-troubleshooting", + "title": "Troubleshooting prisma errors" + }, + { + "url": "https://supabase.com/docs/guides/platform/read-replicas/getting-started", + "title": "Getting started with Read Replicas" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-retrieve-tables", + "title": "Failed to retrieve tables" } ], - "resultChars": 28992 + "resultChars": 29720 } ] }, @@ -663,7 +613,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019fd0c3-00c0-723a-ab58-557e43b2c3c5/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019fd5ea-0355-7642-9590-09c3baf6e8d1/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -672,7 +622,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "Diagnosed missing owner-scoped UPDATE policy for storage.objects on avatars upsert, kept public read/RLS, and added authenticated UPDATE policy with USING and WITH CHECK scoped to auth.uid()." + "judgeNotes": "The assistant correctly diagnosed missing storage.objects UPDATE RLS policy for avatar replacement/upsert, explained public bucket only affects read/download, added an authenticated owner-scoped UPDATE policy with both USING and WITH CHECK, and kept the bucket public with RLS enabled." } ], "skills": { @@ -731,17 +681,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"da11b843-213f-424b-ab13-40665802366d\",\"metric\":\"steps_a_msfr51fx\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"7208214b-972f-4df9-a125-c87b1dae7ebd\",\"metric\":\"steps_a_msh6kumt\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"da11b843-213f-424b-ab13-40665802366d\",\"metric\":\"steps_a_msfr51fx\",\"value\":111}]" + "notes": "status 200: [{\"user_id\":\"7208214b-972f-4df9-a125-c87b1dae7ebd\",\"metric\":\"steps_a_msh6kumt\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"fd192dff-6f33-4a01-ba72-5cb02ddc5d80\",\"metric\":\"steps_b_msfr51fx\",\"value\":222}]" + "notes": "status 200: [{\"user_id\":\"c3212823-31ce-434f-a3e2-c6d587762597\",\"metric\":\"steps_b_msh6kumt\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -805,7 +755,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The assistant correctly treats the task as Supabase Realtime/Postgres Changes setup, adds the messages table to the supabase_realtime publication, provides a postgres_changes client subscription, and does not recommend or confuse read replicas with Realtime." + "judgeNotes": "The assistant correctly treated the issue as Supabase Realtime/Postgres Changes setup, added the messages table to the supabase_realtime publication, verified it, and provided a postgres_changes client subscription. It did not recommend or imply read replicas are needed or useful for live updates." } ], "skills": { @@ -843,17 +793,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified `video-thumbnails` as affected and explicitly noted status_code 546 due to CPU/resource limit, not 500/503." + "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly described failures as HTTP 546 due to CPU/resource limit, not 500/503." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence: `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)` and explains it is not wall-clock, memory, exception, or unrelated errors." + "judgeNotes": "Assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence: shutdown reason CPUTime and cpu_time_used/cpu_time_limit at 2000ms. It explicitly rules out code exception and unrelated functions." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "Recommended moving CPU-heavy thumbnail generation out of the edge function and reducing per-call CPU cost, directly addressing the fixed CPU limit." + "judgeNotes": "The assistant clearly recommends reducing per-call CPU usage, moving thumbnail generation to a background worker/container, and cutting the work per invocation." } ], "skills": { @@ -935,7 +885,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "The answer correctly diagnoses secure-by-default missing table grants, distinguishes grants from RLS, preserves existing owner-scoped RLS, grants only SELECT/INSERT on public.journal_entries to authenticated, and does not weaken RLS or grant anon/public." + "judgeNotes": "Diagnoses missing Data API/table grants separately from RLS, preserves owner-scoped RLS, and grants only SELECT/INSERT on public.journal_entries to authenticated." } ], "skills": { @@ -947,7 +897,7 @@ }, "prompt": "Our app lets signed-in users keep a private journal. Entries are stored in the `journal_entries` table. A user should only ever see their own, and they need to be able to create new entries from the app.\n\nI can see the rows when I query the table directly, but the journal page shows an empty list for every signed-in user. New entries don't show up after saving either. Please figure out why and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-secure-default-grants/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-dataapi-002-secure-default-grants.json" }, { @@ -978,7 +928,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"8945134f-1830-47ff-b927-da6ca2515a7e\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"78e9459a-d3b9-4554-a4f1-0bc176e34531\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -987,7 +937,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "The assistant correctly identified the missing USING clause on the UPDATE RLS policy as the cause of silent 0-row updates, explained USING vs WITH CHECK, and fixed the policy for authenticated users with USING (user_id = auth.uid()) while retaining WITH CHECK (user_id = auth.uid()) and RLS isolation." + "judgeNotes": "The answer correctly identifies the missing USING clause on the UPDATE RLS policy as the cause of silent zero-row updates, explains WITH CHECK vs USING, and fixes it with USING (user_id = auth.uid()) while preserving WITH CHECK (user_id = auth.uid()) and authenticated scoping." } ], "skills": { @@ -998,31 +948,19 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"RLS policy UPDATE USING WITH CHECK default\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"RLS policy UPDATE USING WITH CHECK default\", limit: 3) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", - "title": "RLS Simplified" } ], - "resultChars": 68480 + "resultChars": 49282 } ] }, @@ -1054,7 +992,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer explicitly says pause/restore is not first-line, distinguishes restart from restore, recommends restart as the safer recovery step for true unhealthy services, and suggests concrete checks like cache/status page plus logs/advisors before acting." + "judgeNotes": "The answer correctly recommends restart as the first recovery step, distinguishes restart from pause/restore, and notes overload/compute tuning if the issue recurs. It does not present pause/restore as the right or reliable recovery path, though it could have more explicitly suggested logs/advisors and contacting support." } ], "skills": { @@ -1062,11 +1000,60 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"project unhealthy restart pause restore unresponsive\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + } + ], + "resultChars": 66555 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"dashboard unresponsive cannot access project restart via API\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/api/v1-restart-a-project", + "title": "Restarts the given project" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-retrieve-tables", + "title": "Failed to retrieve tables" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/supabase-dashboard-not-loading-project-not-loading-on-dashboard-LfMq9F", + "title": "Supabase dashboard not loading/ Project not loading on dashboard" + } + ], + "resultChars": 4529 + } + ] }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", "promptSourcePath": "evals/resolve-reliability-001-unhealthy-project-recovery/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, { @@ -1110,7 +1097,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019fd0c2-cea2-748d-b150-598dc0432644/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019fd5e9-b73d-751f-a923-21301ad5ba3a/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -1119,7 +1106,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert replacements, kept public-read/RLS, and added authenticated owner-scoped UPDATE policy with USING and WITH CHECK." + "judgeNotes": "Diagnosed missing UPDATE policy for upsert replacement, kept public-read/RLS setup intact, and added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK." } ], "skills": {