diff --git a/.gitignore b/.gitignore index 932a6aea..5dc75e68 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,12 @@ node_modules/ dist/ .env .env*.local +# eval seed data may include a .env with well-known local demo keys +!evals/resolve-sdk-001-legacy-key-migration/local/app/.env + +# generated local CLI state, not eval seed data +evals/*/local/supabase/.temp/ +evals/*/local/supabase/.branches/ .DS_Store results/*/ .sync-tmp/ diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 5b28a57e..2371d4c9 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1,4 +1,334 @@ [ + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "error 42501: permission denied for table todos" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "List any entries tagged breaking-change, and anything relevant to: local development with the CLI, database migrations, RLS policies, the anon/authenticated roles, API keys (publishable/secret vs anon/service_role), seed files, or config.toml. Include dates and links.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 8255 + }, + { + "source": "web_fetch", + "query": "Show the full recommended workflow and exact SQL for exposing a table in the public schema to the Data API, including GRANT statements for anon and authenticated roles, enabling RLS, and read-only access patterns. Quote the SQL verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api.md" + } + ], + "resultChars": 1245 + } + ] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restore pg_dump custom format dump into Supabase migrate existing postgres database\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + } + ], + "resultChars": 78095 + }, + { + "source": "web_fetch", + "query": "What are the exact recommended commands and flags for restoring a pg_dump dump into a Supabase Postgres database? Include any notes about roles, ownership, privileges, extensions, schemas to exclude, and disabling triggers.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" + } + ], + "resultChars": 1424 + }, + { + "source": "web_fetch", + "query": "List any recent entries tagged breaking-change, especially anything related to the CLI, local development, `supabase start`, `supabase db` commands, database restores/migrations, or Postgres major versions.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1248 + } + ] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-cli-001-bootstrap-app", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "migrations", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, { "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", @@ -8,50 +338,60 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-auth-001-email-password-flow", "stage": "build", "product": [ - "database", - "data-api" + "auth", + "database" ], "topic": [ - "migrations", + "sdk", "rls" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" }, { - "name": "todos table is created by a migration file", - "passed": true + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user d720a18b-4270-462b-bc8d-62f6f9792f54, signUp returned {\"userId\":\"d720a18b-4270-462b-bc8d-62f6f9792f54\"}" }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "signup metadata reaches the profile (display name)", "passed": true, - "notes": "found 2 rows" + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"d720a18b-4270-462b-bc8d-62f6f9792f54\"}" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "getMyProfile returns the signed-in user's profile", "passed": true, - "notes": "error 42501: permission denied for table todos" + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "app code does not use the secret / service-role key", "passed": true, - "notes": "2 rows" + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -66,33 +406,53 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and anything relevant to: local development with the CLI, database migrations, RLS policies, the anon/authenticated roles, API keys (publishable/secret vs anon/service_role), seed files, or config.toml. Include dates and links.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -60", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 8255 + "resultChars": 3161 }, { - "source": "web_fetch", - "query": "Show the full recommended workflow and exact SQL for exposing a table in the public schema to the Data API, including GRANT statements for anon and authenticated roles, enabling RLS, and read-only access patterns. Quote the SQL verbatim.", + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -n -i -B2 'Breaking Change · .*\\(Auth\\|auth\\)' | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1353 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"signUp email password options data user metadata javascript\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api.md" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" } ], - "resultChars": 1245 + "resultChars": 22309 } ] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-opus-5/build-auth-001-email-password-flow.json" }, { "experiment": "claude-code-opus-5", @@ -304,37 +664,44 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -350,64 +717,103 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"restore pg_dump custom format dump into Supabase migrate existing postgres database\", limit: 6) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"secret key service_role server-side createClient node backend persistSession\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", - "title": "Migrate from Heroku to Supabase" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 78095 - }, - { - "source": "web_fetch", - "query": "What are the exact recommended commands and flags for restoring a pg_dump dump into a Supabase Postgres database? Include any notes about roles, ownership, privileges, extensions, schemas to exclude, and disabling triggers.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" - } - ], - "resultChars": 1424 - }, - { - "source": "web_fetch", - "query": "List any recent entries tagged breaking-change, especially anything related to the CLI, local development, `supabase start`, `supabase db` commands, database restores/migrations, or Postgres major versions.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" } ], - "resultChars": 1248 + "resultChars": 45578 } ] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-opus-5/build-dataapi-001-relational-report.json" + }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-opus-5", @@ -2255,50 +2661,60 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-auth-001-email-password-flow", "stage": "build", "product": [ - "database", - "data-api" + "auth", + "database" ], "topic": [ - "migrations", + "sdk", "rls" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" }, { - "name": "todos table is created by a migration file", - "passed": true + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 6b4b33dc-03b1-440f-b7d8-0c2eaf088d50, signUp returned {\"userId\":\"6b4b33dc-03b1-440f-b7d8-0c2eaf088d50\"}" }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "signup metadata reaches the profile (display name)", "passed": true, - "notes": "found 2 rows" + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"6b4b33dc-03b1-440f-b7d8-0c2eaf088d50\"}" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "getMyProfile returns the signed-in user's profile", "passed": true, - "notes": "0 rows" + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "app code does not use the secret / service-role key", "passed": true, - "notes": "2 rows" + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -2308,10 +2724,10 @@ "docs": { "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-opus-5-no-skills/build-auth-001-email-password-flow.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -2480,37 +2896,105 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -2520,10 +3004,10 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -3671,6 +4155,88 @@ "attempts": 1, "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 00a4b125-5d0f-4e3e-96a8-7df8041aa694, signUp returned {\"userId\":\"00a4b125-5d0f-4e3e-96a8-7df8041aa694\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"00a4b125-5d0f-4e3e-96a8-7df8041aa694\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -3817,27 +4383,228 @@ "queues" ], "topic": [ - "sql", + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 3) from the queue" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule pgmq queue send message example\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", + "title": "pg_cron debugging guide" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + } + ], + "resultChars": 55834 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"edge function supabase-js service role client import example Deno.serve\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/unit-test", + "title": "Testing your Edge Functions" + } + ], + "resultChars": 26350 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Edge Functions default secrets SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY automatically populated local\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + } + ], + "resultChars": 28165 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "name": "report runs and prints JSON", "passed": true, - "notes": "schedule='* * * * *', active=true" + "notes": "exit 0" }, { - "name": "cron command enqueues to the 'tasks' queue", + "name": "alerts match the database (below threshold, sorted)", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "process-tasks function drains the queue", + "name": "tables stay locked down (publishable key reads nothing)", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -3845,86 +4612,15 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule pgmq queue send message example\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/pgcron-debugging-guide-n1KTaz", - "title": "pg_cron debugging guide" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - } - ], - "resultChars": 55834 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function supabase-js service role client import example Deno.serve\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions/unit-test", - "title": "Testing your Edge Functions" - } - ], - "resultChars": 26350 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Edge Functions default secrets SUPABASE_URL SUPABASE_SERVICE_ROLE_KEY automatically populated local\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" - } - ], - "resultChars": 28165 - } - ] + "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-sonnet-5", @@ -5699,6 +6395,83 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user af7cdf33-fe1f-4c8a-9285-3691de4a9062, signUp returned {\"userId\":\"af7cdf33-fe1f-4c8a-9285-3691de4a9062\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"af7cdf33-fe1f-4c8a-9285-3691de4a9062\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -5789,20 +6562,134 @@ "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": true + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", + "passed": true + }, + { + "name": "description column exists in the live database", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Add a description text column to the `products` table in my local Supabase stack", + "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-cli-003-pg-cron-queue-workflow", + "stage": "build", + "product": [ + "database", + "edge-functions", + "cron", + "queues" + ], + "topic": [ + "sql", + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "passed": true, + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 3) from the queue" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "schema file updated to include description column", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "a new migration was generated for the change", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" }, { - "name": "description column exists in the live database", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -5812,10 +6699,10 @@ "docs": { "calls": [] }, - "prompt": "Add a description text column to the `products` table in my local Supabase stack", - "promptSourcePath": "evals/build-cli-002-declarative-schema/PROMPT.md", + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-002-declarative-schema.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -5826,36 +6713,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-cli-003-pg-cron-queue-workflow", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ - "database", - "edge-functions", - "cron", - "queues" + "data-api", + "database" ], "topic": [ - "sql", "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", + "name": "report runs and prints JSON", "passed": true, - "notes": "schedule='* * * * *', active=true" + "notes": "exit 0" }, { - "name": "cron command enqueues to the 'tasks' queue", + "name": "alerts match the database (below threshold, sorted)", "passed": true, - "notes": "queue depth 0 -> 1" + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "process-tasks function drains the queue", + "name": "tables stay locked down (publishable key reads nothing)", "passed": true, - "notes": "function removed the seeded message (id 3) from the queue" + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -5865,10 +6760,10 @@ "docs": { "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6910,6 +7805,160 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user eae71afb-4f4b-42c1-9749-cf6007ba7c10, signUp returned {\"userId\":\"eae71afb-4f4b-42c1-9749-cf6007ba7c10\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"eae71afb-4f4b-42c1-9749-cf6007ba7c10\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession authenticated profile table select\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + } + ], + "resultChars": 116820 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"site: supabase.com/docs/reference/javascript auth signUp signInWithPassword user_metadata display_name\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", + "title": "Identity Linking" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/astrojs", + "title": "Use Supabase Auth with Astro" + } + ], + "resultChars": 186803 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -7205,10 +8254,144 @@ } ] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -9501,6 +10684,83 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 31f642a8-b8cb-4a51-9506-fd7e53e58b4f, signUp returned {\"userId\":\"31f642a8-b8cb-4a51-9506-fd7e53e58b4f\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"31f642a8-b8cb-4a51-9506-fd7e53e58b4f\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -9708,6 +10968,128 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -10953,10 +12335,140 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user ed685aae-6c81-43fd-b019-bcf3b9bfc3a5, signUp returned {\"userId\":\"ed685aae-6c81-43fd-b019-bcf3b9bfc3a5\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"ed685aae-6c81-43fd-b019-bcf3b9bfc3a5\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data user metadata signInWithPassword getUser profile table RLS auth.uid trigger new user\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/swift/auth-signup", + "title": "signUp()" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signup" + } + ], + "resultChars": 24252 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG AUTH HITS ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|auth|signup|sign.?in' | head -n 120 || true; printf '%s\\\\n' '--- WORKSPACE FILES ---'; rg --files -g '\"'!node_modules'\"' | sort; printf '%s\\\\n' '--- AUTH ---'; sed -n '1,240p' app/src/auth.mjs; printf '%s\\\\n' '--- PACKAGE ---'; cat app/package.json; printf '%s\\\\n' '--- SUPABASE CONFIG ---'; sed -n '1,260p' supabase/config.toml; printf '%s\\\\n' '--- MIGRATIONS ---'; find supabase -maxdepth 3 -type f -print -exec sh -c 'case \\\"\"'$1\" in *.sql) echo \"--- $1\"; sed -n \"1,260p\" \"$1\";; esac'\"' sh {} \\\\;; printf '%s\\\\n' '--- STATUS ---'; supabase status\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1467 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" }, { "experiment": "codex-gpt-5.6", @@ -10965,7 +12477,7 @@ "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" + "reasoningEffort": "low" }, "eval": "build-cli-001-bootstrap-app", "stage": "build", @@ -11779,6 +13291,214 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js select nested relationships foreign key alias service role server Node\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + }, + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 47636 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '---CHANGELOG---'; curl -fsSL https://supabase.com/changelog.md | head -n 120; printf '%s\\\\n' '---AGENTS---'; find .. -name AGENTS.md -print; printf '%s\\\\n' '---REPORT---'; sed -n '1,260p' app/report.mjs; printf '%s\\\\n' '---PACKAGE---'; for f in package.json app/package.json; do if [ -f \\\"\"'$f\" ]; then echo \"### $f\"; cat \"$f\"; fi; done; printf '\"'%s\\\\n' '---SUPABASE FILES---'; find supabase -maxdepth 3 -type f \"'! -path '\"'*/.temp/*' -print | sort; printf '%s\\\\n' '---SQL---'; for f in \"'$(find supabase -maxdepth 3 -type f '\"\\\\( -name '*.sql' -o -name 'config.toml' \\\\) \"'! -path '\"'*/.temp/*' | sort); do echo \\\"### \"'$f\"; sed -n '\"'1,300p' \\\"\"'$f\"; done'", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6428 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/restock.mjs ---' && sed -n '1,240p' app/restock.mjs && printf '%s\\\\n' '--- app/package.json ---' && cat app/package.json && printf '%s\\\\n' '--- migration ---' && sed -n '1,320p' supabase/migrations/0000_inventory_schema.sql && printf '%s\\\\n' '--- config ---' && sed -n '1,220p' supabase/config.toml && printf '%s\\\\n' '--- status ---' && supabase status && printf '%s\\\\n' '--- changelog relevant lines ---' && curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|postgrest|supabase-js|javascript'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5661 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase JavaScript select foreign tables nested relationships createClient secret key backend Node\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + } + ], + "resultChars": 24344 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ @@ -13527,26 +15247,176 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security multi tenant team workspace membership policies auth.uid() security definer\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"row level security multi tenant team workspace membership policies auth.uid() security definer\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" + } + ], + "resultChars": 38609 + } + ] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user c8115185-6cf0-4a18-9a3f-b354212375ca, signUp returned {\"userId\":\"c8115185-6cf0-4a18-9a3f-b354212375ca\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"c8115185-6cf0-4a18-9a3f-b354212375ca\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient signUp email password user metadata signInWithPassword getUser select single profiles browser client\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" + } + ], + "resultChars": 130679 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript auth signUp options data metadata signInWithPassword select maybeSingle\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithsso" + }, + { + "url": "https://supabase.com/docs/reference/dart/using-modifiers-maybesingle" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" } ], - "resultChars": 38609 + "resultChars": 19708 } ] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" }, { "experiment": "codex-gpt-5.6-no-skills", @@ -13555,7 +15425,7 @@ "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" + "reasoningEffort": "low" }, "eval": "build-cli-001-bootstrap-app", "stage": "build", @@ -14327,6 +16197,186 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js Node select nested relationships service role key local development\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + } + ], + "resultChars": 67379 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js select foreign table relationships environment variables service role Node.js\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", + "title": "Vercel Marketplace" + } + ], + "resultChars": 70905 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ @@ -15865,10 +17915,91 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 78735a72-891a-4831-b74d-a7eacd855314, signUp returned {\"userId\":\"78735a72-891a-4831-b74d-a7eacd855314\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"78735a72-891a-4831-b74d-a7eacd855314\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "opencode-kimi-k3/build-auth-001-email-password-flow.json" }, { "experiment": "opencode-kimi-k3", @@ -16131,6 +18262,136 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3/build-dataapi-001-relational-report.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "opencode-kimi-k3", "experimentSuite": "benchmark", @@ -17542,6 +19803,82 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 2629a92e-db2f-495e-b419-0010702fc708, signUp returned {\"userId\":\"2629a92e-db2f-495e-b419-0010702fc708\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"2629a92e-db2f-495e-b419-0010702fc708\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -17739,6 +20076,126 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 97c715da..b3c61b73 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -622,6 +622,78 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" + }, + { + "name": "posts script actually depends on its publishable key", + "passed": false, + "notes": "still produced [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"] with a corrupted publishable key — looks hardcoded or key-agnostic" + }, + { + "name": "stats script actually depends on its secret key", + "passed": false, + "notes": "still produced {\"drafts\":0} with a corrupted secret key — looks hardcoded or key-agnostic" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -1069,6 +1141,73 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" + }, + { + "name": "posts script actually depends on its publishable key", + "passed": false, + "notes": "still produced [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"] with a corrupted publishable key — looks hardcoded or key-agnostic" + }, + { + "name": "stats script actually depends on its secret key", + "passed": false, + "notes": "still produced {\"drafts\":0} with a corrupted secret key — looks hardcoded or key-agnostic" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", diff --git a/evals/build-auth-001-email-password-flow/EVAL.ts b/evals/build-auth-001-email-password-flow/EVAL.ts new file mode 100644 index 00000000..64b9c878 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/EVAL.ts @@ -0,0 +1,264 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Email+password auth benchmark: the prompt asks the agent to finish the +// app's auth layer (app/src/auth.mjs) against the running local stack and +// never names supabase-js — the "uses @supabase/supabase-js" check is +// GATING. On top of SDK discovery it measures driving auth correctly: +// passing the display name as signup user metadata (the seeded profiles +// trigger reads raw_user_meta_data), handling bad credentials gracefully, +// and reading the RLS-scoped profile with the session actually attached. + +const APP_DIR = 'app'; +const DRIVER = 'eval-driver.mjs'; +const DRIVER_MARKER = '___EVAL_DRIVER___'; + +// Runs inside the sandbox, in one process, exactly like the app would use +// the module: sign up, fail a sign-in, sign in, read the profile. +const DRIVER_SOURCE = ` +import { signUp, signIn, getMyProfile } from './src/auth.mjs'; + +const [email, password, displayName, wrongPassword] = process.argv.slice(2); +const out = {}; +const step = async (name, fn) => { + try { + out[name] = await fn(); + } catch (error) { + out[name] = { + threw: String(error instanceof Error ? error.message : error), + }; + } +}; +await step('signUp', () => signUp(email, password, displayName)); +await step('signInWrong', () => signIn(email, wrongPassword)); +await step('signIn', () => signIn(email, password)); +await step('profile', () => getMyProfile()); +console.log('${DRIVER_MARKER}' + JSON.stringify(out)); +process.exit(0); +`; + +interface DriverStep { + userId?: unknown; + displayName?: unknown; + plan?: unknown; + error?: unknown; + threw?: unknown; +} + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + // Unique suffix keeps signup emails collision-free across attempts. + const suffix = Date.now().toString(36); + const email = `alex-${suffix}@example.com`; + const password = 'correct-horse-battery'; + const wrongPassword = 'wrong-horse-battery'; + const displayName = 'Alex Doe'; + + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const publishableKey = str(status.PUBLISHABLE_KEY); + if (!apiUrl || !publishableKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/PUBLISHABLE_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about auth, + // not npm. A no-op when the agent already installed dependencies. + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, + { timeoutMs: 180_000 } + ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } + + const written = await writeDriver(ctx); + if (!written.ok) { + return fail( + 'installed the eval driver', + written.stderr.trim() || written.stdout.trim() + ); + } + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_PUBLISHABLE_KEY="${publishableKey}" ` + + `node ${DRIVER} "${email}" "${password}" "${displayName}" "${wrongPassword}"`, + { timeoutMs: 60_000 } + ); + const out = parseDriverOutput(run.stdout); + checks.push({ + name: 'auth module loads and the driver completes', + passed: out !== undefined, + notes: out + ? 'driver produced a result' + : `no driver output — ${preview(run.stderr || run.stdout)}`, + }); + + const signUp = (out?.signUp ?? {}) as DriverStep; + const signInWrong = (out?.signInWrong ?? {}) as DriverStep; + const signIn = (out?.signIn ?? {}) as DriverStep; + const profile = (out?.profile ?? {}) as DriverStep; + + // Ground truth from the database (superuser query bypasses RLS). + const { rows: userRows } = await ctx.query( + `select u.id::text as id, p.display_name, p.plan + from auth.users u + left join public.profiles p on p.id = u.id + where u.email = '${email}'` + ); + const dbUser = userRows[0]; + + checks.push({ + name: 'signUp creates the account and returns its user id', + passed: !!dbUser && !!signUp.userId && signUp.userId === dbUser.id, + notes: dbUser + ? `db user ${dbUser.id}, signUp returned ${JSON.stringify(signUp)}` + : 'no auth.users row for the signup email', + }); + + // The seeded trigger falls back to the email local part, so the real + // display name only arrives if signUp sent it as user metadata. + checks.push({ + name: 'signup metadata reaches the profile (display name)', + passed: dbUser?.display_name === displayName, + notes: `profiles.display_name = ${JSON.stringify(dbUser?.display_name ?? null)}`, + }); + + checks.push({ + name: 'wrong password is rejected gracefully (no throw, no session)', + passed: !!signInWrong.error && !signInWrong.userId && !signInWrong.threw, + notes: JSON.stringify(signInWrong), + }); + + checks.push({ + name: 'signIn with the right password returns the user id', + passed: !!dbUser && signIn.userId === dbUser.id, + notes: JSON.stringify(signIn), + }); + + checks.push({ + name: "getMyProfile returns the signed-in user's profile", + passed: profile.displayName === displayName && profile.plan === 'free', + notes: JSON.stringify(profile), + }); + + // Client-side code: RLS + the publishable key are enough; the secret / + // service-role key must not appear anywhere in the app. + const secretScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules 'sb_secret_|SERVICE_ROLE' ${APP_DIR} || true` + ); + checks.push({ + name: 'app code does not use the secret / service-role key', + passed: secretScan.stdout.trim() === '', + notes: secretScan.stdout.trim() || 'no secret-key references found', + }); + + // GATING: the auth layer must be built on supabase-js, even though the + // prompt never names it. + checks.push(await sdkUsageCheck(ctx)); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function writeDriver(ctx: LocalStackEvalContext) { + const encoded = Buffer.from(DRIVER_SOURCE, 'utf-8').toString('base64'); + return ctx.exec(`echo ${encoded} | base64 -d > ${APP_DIR}/${DRIVER}`); +} + +function parseDriverOutput( + stdout: string +): Record | undefined { + const line = stdout + .split('\n') + .find((candidate) => candidate.includes(DRIVER_MARKER)); + if (!line) return undefined; + try { + return JSON.parse( + line.slice(line.indexOf(DRIVER_MARKER) + DRIVER_MARKER.length) + ); + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * the specifier must be closed by a matching quote (so `-not-real` doesn't + * match) and sit on a `from`/`require(`/`import(` line that isn't commented + * out. Multi-line named imports still match: the closing `} from '…'` line + * always carries `from` alongside the specifier. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rnE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"(from|require\\(|import\\()\\s*['\\"](npm:)?@supabase/supabase-js['\\"]" ${APP_DIR} ` + + `| grep -vE ':[0-9]+:\\s*(//|\\*)' || true` + ); + const files = [ + ...new Set( + scan.stdout + .trim() + .split('\n') + .filter(Boolean) + .map((line) => line.slice(0, line.indexOf(':'))) + ), + ]; + return { + name: NAME, + passed: files.length > 0, + notes: + files.length > 0 + ? `imports found in: ${files.join(', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-auth-001-email-password-flow/PROMPT.md b/evals/build-auth-001-email-password-flow/PROMPT.md new file mode 100644 index 00000000..9061dada --- /dev/null +++ b/evals/build-auth-001-email-password-flow/PROMPT.md @@ -0,0 +1,31 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - auth + - database +topic: + - sdk + - rls +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + The signup → profile-trigger flow is a recurring pain point + (supabase/supabase#37497, supabase/supabase#35997, and the canonical + pattern in https://supabase.com/docs/guides/auth/managing-user-data); + auth is also the largest supabase-js surface with no dedicated eval + coverage. +--- + +Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in +there describe what each function should do. People sign up with an email, +password, and display name, sign back in later, and the app greets them with +their profile. + +The Supabase project for this app is in `supabase/` and already running +locally. When you're done, the functions should work for real against it. diff --git a/evals/build-auth-001-email-password-flow/README.md b/evals/build-auth-001-email-password-flow/README.md new file mode 100644 index 00000000..5a238bf0 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/README.md @@ -0,0 +1,19 @@ +# build-auth-001-email-password-flow + +Benchmark for supabase-js auth flows. The prompt is a casual "our app needs +accounts" ask pointing at stubbed functions in `local/app/src/auth.mjs` +(signUp / signIn / getMyProfile) — it never names supabase-js; the +"uses @supabase/supabase-js" check is GATING, like its precedent in +build-functions-005. + +The seed teaches through data rather than the prompt: the `profiles` trigger +falls back to the email local part unless the signup sends `display_name` +as user metadata, so the "display name reaches the profile" check only +passes when the agent wires `signUp` with `options.data`. RLS on `profiles` +plus the publishable key make the client-side path the only sanctioned one; +a "no secret key in app code" check guards the boundary. + +Scoring installs a small driver (`app/eval-driver.mjs`) that imports the +agent's module and exercises the contract in one process — sign up, wrong +password (must not throw), correct sign-in, profile read — then verifies +results against the database with the superuser connection. diff --git a/evals/build-auth-001-email-password-flow/local/app/package.json b/evals/build-auth-001-email-password-flow/local/app/package.json new file mode 100644 index 00000000..ccbe7c7e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "acme-app", + "private": true, + "type": "module" +} diff --git a/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs new file mode 100644 index 00000000..2e312295 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs @@ -0,0 +1,36 @@ +// Auth layer for the app. The rest of the app calls these three functions; +// wire them up to our Supabase project (it's running locally — see +// ../../supabase). Connection settings come from the environment: +// SUPABASE_URL and SUPABASE_PUBLISHABLE_KEY. +// +// This module is used from client-side code, so it must only ever hold the +// publishable (client) key. + +/** + * Create an account with email + password. `displayName` should end up as + * the user's profile display name. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure. + */ +export async function signUp(email, password, displayName) { + throw new Error('TODO: implement signUp'); +} + +/** + * Sign in with email + password. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure + * (e.g. wrong password) — it must not throw for bad credentials. + */ +export async function signIn(email, password) { + throw new Error('TODO: implement signIn'); +} + +/** + * The currently signed-in user's profile from the `profiles` table, as + * `{ displayName, plan }`. Resolves to `{ error: string }` when nobody is + * signed in. + */ +export async function getMyProfile() { + throw new Error('TODO: implement getMyProfile'); +} diff --git a/evals/build-auth-001-email-password-flow/local/supabase/config.toml b/evals/build-auth-001-email-password-flow/local/supabase/config.toml new file mode 100644 index 00000000..275fb4b8 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-auth-flow" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql new file mode 100644 index 00000000..b0eec56e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql @@ -0,0 +1,53 @@ +-- Public profile for each account, created automatically on signup. Guarded +-- by row-level security so users can only see and edit their own profile. +create table public.profiles ( + id uuid primary key references auth.users (id) on delete cascade, + display_name text not null, + plan text not null default 'free', + created_at timestamptz not null default now() +); + +alter table public.profiles enable row level security; + +create policy "users can read their own profile" + on public.profiles + for select + to authenticated + using (auth.uid() = id); + +create policy "users can update their own profile" + on public.profiles + for update + to authenticated + using (auth.uid() = id) + with check (auth.uid() = id); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows. +grant select, update on public.profiles to authenticated; + +-- Create the profile row when a user signs up. The display name comes from +-- the signup user metadata; if the app doesn't send one, we fall back to the +-- email local part. +create function public.handle_new_user() +returns trigger +language plpgsql +security definer +set search_path = '' +as $$ +begin + insert into public.profiles (id, display_name) + values ( + new.id, + coalesce( + new.raw_user_meta_data ->> 'display_name', + split_part(new.email, '@', 1) + ) + ); + return new; +end; +$$; + +create trigger on_auth_user_created + after insert on auth.users + for each row execute function public.handle_new_user(); diff --git a/evals/build-dataapi-001-relational-report/EVAL.ts b/evals/build-dataapi-001-relational-report/EVAL.ts new file mode 100644 index 00000000..62ea25ca --- /dev/null +++ b/evals/build-dataapi-001-relational-report/EVAL.ts @@ -0,0 +1,233 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Relational-report benchmark for the Data API: the prompt asks the agent to +// finish a backend reporting script over a seeded relational schema +// (customers → orders → order_items → products) and never names supabase-js — +// the "uses @supabase/supabase-js" check is GATING, and shelling out to +// psql / a raw Postgres driver instead fails. Expected numbers are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'report.mjs'; + +interface ReportRow { + customer: unknown; + orderCount: unknown; + totalCents: unknown; + topProduct: unknown; +} + +const EXPECTED_SQL = ` +with per_product as ( + select c.name as customer, p.name as product, + sum(oi.quantity)::int as units + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name, p.name +), totals as ( + select c.name as customer, + count(distinct o.id)::int as order_count, + sum(oi.quantity * p.price_cents)::int as total_cents + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name +) +select t.customer, + t.order_count, + t.total_cents, + (select pp.product + from per_product pp + where pp.customer = t.customer + order by pp.units desc, pp.product asc + limit 1) as top_product + from totals t + order by t.customer asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, + { timeoutMs: 180_000 } + ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: run.ok && actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + customer: row.customer, + orderCount: row.order_count, + totalCents: row.total_cents, + topProduct: row.top_product, + })); + const normalized = (actual ?? []).map((row) => ({ + customer: row.customer, + orderCount: row.orderCount, + totalCents: row.totalCents, + topProduct: row.topProduct, + })); + checks.push({ + name: 'report numbers match the database (per customer, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('customers').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): ReportRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as ReportRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * the specifier must be closed by a matching quote (so `-not-real` doesn't + * match) and sit on a `from`/`require(`/`import(` line that isn't commented + * out. Multi-line named imports still match: the closing `} from '…'` line + * always carries `from` alongside the specifier. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rnE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"(from|require\\(|import\\()\\s*['\\"](npm:)?@supabase/supabase-js['\\"]" ${APP_DIR} ` + + `| grep -vE ':[0-9]+:\\s*(//|\\*)' || true` + ); + const files = [ + ...new Set( + scan.stdout + .trim() + .split('\n') + .filter(Boolean) + .map((line) => line.slice(0, line.indexOf(':'))) + ), + ]; + return { + name: NAME, + passed: files.length > 0, + notes: + files.length > 0 + ? `imports found in: ${files.join(', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-001-relational-report/PROMPT.md b/evals/build-dataapi-001-relational-report/PROMPT.md new file mode 100644 index 00000000..00957811 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/PROMPT.md @@ -0,0 +1,28 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + Relationship embedding is the query-builder surface users trip on most + (supabase/postgrest-js#609, supabase/postgrest-js#611, + supabase/supabase-js#1639), and the most-used supabase-js surface had no + dedicated eval coverage. +--- + +We need the nightly sales report working. `app/report.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON summary of what +each customer has ordered. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right numbers. diff --git a/evals/build-dataapi-001-relational-report/local/app/package.json b/evals/build-dataapi-001-relational-report/local/app/package.json new file mode 100644 index 00000000..81652eb0 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "reporting-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-001-relational-report/local/app/report.mjs b/evals/build-dataapi-001-relational-report/local/app/report.mjs new file mode 100644 index 00000000..757366f8 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/report.mjs @@ -0,0 +1,22 @@ +// Nightly sales report, run inside our Node backend worker: +// +// node report.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per customer who has placed at +// least one order, sorted by customer name: +// +// { +// "customer": string, // customer name +// "orderCount": number, // how many orders they placed +// "totalCents": number, // total spent across all their orders +// "topProduct": string // product they bought the most units of +// } +// +// If two products tie on units, topProduct is the alphabetically first one. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-001-relational-report/local/supabase/config.toml b/evals/build-dataapi-001-relational-report/local/supabase/config.toml new file mode 100644 index 00000000..8ca57d9f --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-orders-report" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql new file mode 100644 index 00000000..29f889c1 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql @@ -0,0 +1,69 @@ +-- Order history for the reporting worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.customers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + price_cents int not null +); + +create table public.orders ( + id bigint generated always as identity primary key, + customer_id bigint not null references public.customers (id), + ordered_at timestamptz not null default now() +); + +create table public.order_items ( + id bigint generated always as identity primary key, + order_id bigint not null references public.orders (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity > 0) +); + +alter table public.customers enable row level security; +alter table public.products enable row level security; +alter table public.orders enable row level security; +alter table public.order_items enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.customers, public.products, public.orders, + public.order_items to service_role; + +-- Seed data. Alan has no orders and must not appear in the report. +insert into public.customers (name, email) values + ('Ada Lovelace', 'ada@example.com'), + ('Grace Hopper', 'grace@example.com'), + ('Linus Pauling', 'linus@example.com'), + ('Alan Turing', 'alan@example.com'); + +insert into public.products (name, price_cents) values + ('Keyboard', 4500), + ('Mouse', 2500), + ('Monitor', 32000), + ('Cable', 900); + +insert into public.orders (customer_id, ordered_at) values + (1, '2026-06-01T10:00:00Z'), + (1, '2026-06-14T09:30:00Z'), + (2, '2026-06-03T16:20:00Z'), + (2, '2026-06-20T11:05:00Z'), + (3, '2026-06-08T14:45:00Z'); + +insert into public.order_items (order_id, product_id, quantity) values + (1, 1, 2), -- Ada: 2x Keyboard + (1, 4, 1), -- Ada: 1x Cable + (2, 3, 1), -- Ada: 1x Monitor + (3, 2, 3), -- Grace: 3x Mouse + (4, 1, 1), -- Grace: 1x Keyboard + (4, 4, 4), -- Grace: 4x Cable + (5, 3, 2), -- Linus: 2x Monitor + (5, 2, 1); -- Linus: 1x Mouse diff --git a/evals/build-dataapi-002-restock-alert-report/EVAL.ts b/evals/build-dataapi-002-restock-alert-report/EVAL.ts new file mode 100644 index 00000000..352eede5 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/EVAL.ts @@ -0,0 +1,221 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Companion to build-dataapi-001-relational-report: same "unnamed SDK, empty +// package.json, backend worker script" shape, different schema (warehouses → +// inventory → products → suppliers) and aggregation (below-threshold restock +// alert, not a sales rollup). Checks whether the SDK-adoption split found +// there (claude-code 4/4, codex 0/4) generalizes or was specific to that +// prompt. The "uses @supabase/supabase-js" check is GATING, and shelling out +// to psql / a raw Postgres driver instead fails. Expected alerts are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'restock.mjs'; + +interface AlertRow { + warehouse: unknown; + product: unknown; + quantity: unknown; + reorderThreshold: unknown; + supplierEmail: unknown; +} + +const EXPECTED_SQL = ` +select w.name as warehouse, + p.name as product, + i.quantity, + p.reorder_threshold, + s.email as supplier_email + from public.inventory i + join public.warehouses w on w.id = i.warehouse_id + join public.products p on p.id = i.product_id + join public.suppliers s on s.id = p.supplier_id + where i.quantity < p.reorder_threshold + order by w.name asc, p.name asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, + { timeoutMs: 180_000 } + ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: run.ok && actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorder_threshold, + supplierEmail: row.supplier_email, + })); + const normalized = (actual ?? []).map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorderThreshold, + supplierEmail: row.supplierEmail, + })); + checks.push({ + name: 'alerts match the database (below threshold, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('inventory').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): AlertRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as AlertRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * the specifier must be closed by a matching quote (so `-not-real` doesn't + * match) and sit on a `from`/`require(`/`import(` line that isn't commented + * out. Multi-line named imports still match: the closing `} from '…'` line + * always carries `from` alongside the specifier. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rnE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"(from|require\\(|import\\()\\s*['\\"](npm:)?@supabase/supabase-js['\\"]" ${APP_DIR} ` + + `| grep -vE ':[0-9]+:\\s*(//|\\*)' || true` + ); + const files = [ + ...new Set( + scan.stdout + .trim() + .split('\n') + .filter(Boolean) + .map((line) => line.slice(0, line.indexOf(':'))) + ), + ]; + return { + name: NAME, + passed: files.length > 0, + notes: + files.length > 0 + ? `imports found in: ${files.join(', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-002-restock-alert-report/PROMPT.md b/evals/build-dataapi-002-restock-alert-report/PROMPT.md new file mode 100644 index 00000000..9dcd9494 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/PROMPT.md @@ -0,0 +1,37 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + build-dataapi-001-relational-report found every codex variant (0/4) skips + @supabase/supabase-js for a bare backend Data API script, hand-rolling raw + HTTP instead, while every claude-code variant (4/4) reached for it + unprompted. A single scenario isn't enough to tell a real model tendency + from a one-off artifact of that prompt's specific shape — this companion + scenario keeps the same "unnamed SDK, empty package.json, backend worker + script" shape but swaps in an unrelated schema and aggregation (inventory + restock alerts vs. sales report) to check whether the pattern generalizes. + Agents defaulting to the wrong data-access path on a backend task is a + real, documented failure beyond this pair of evals too + (supabase/agent-skills#173: a Codex session reached for raw PostgREST + credentials and an admin browser instead of the intended tool for a data + read/update task). +--- + +Purchasing needs a restock alert. `app/restock.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON list of what +needs reordering, with who to email about it. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right alerts. diff --git a/evals/build-dataapi-002-restock-alert-report/README.md b/evals/build-dataapi-002-restock-alert-report/README.md new file mode 100644 index 00000000..47cc1eb4 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/README.md @@ -0,0 +1,9 @@ +Companion to `build-dataapi-001-relational-report`. Same shape (unnamed SDK, +empty `package.json`, bare backend worker script reading `SUPABASE_URL` / +`SUPABASE_SECRET_KEY` from the env) but a different schema and aggregation +(inventory restock alerts vs. a sales report), to check whether that eval's +SDK-adoption split (claude-code 4/4 vs. codex 0/4) is a real model tendency +or an artifact of that one prompt. Beyond this pair of evals, +supabase/agent-skills#173 documents the same underlying failure shape in the +wild — a Codex session reaching for raw PostgREST credentials and an admin +browser instead of the intended tool for a data task. diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/package.json b/evals/build-dataapi-002-restock-alert-report/local/app/package.json new file mode 100644 index 00000000..4f4a0bce --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "restock-alert-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs new file mode 100644 index 00000000..1d247baf --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs @@ -0,0 +1,24 @@ +// Restock alert worker, run inside our Node backend worker: +// +// node restock.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per warehouse/product +// combination that's below its reorder threshold, sorted by warehouse name +// then product name: +// +// { +// "warehouse": string, // warehouse name +// "product": string, // product name +// "quantity": number, // current quantity on hand +// "reorderThreshold": number, // reorder threshold for this product +// "supplierEmail": string // email of the product's supplier +// } +// +// Only include rows where quantity is strictly below the reorder threshold. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml new file mode 100644 index 00000000..06330f25 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-restock-alert" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql new file mode 100644 index 00000000..0dfdff73 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql @@ -0,0 +1,68 @@ +-- Inventory for the restock-alert worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.warehouses ( + id bigint generated always as identity primary key, + name text not null +); + +create table public.suppliers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + supplier_id bigint not null references public.suppliers (id), + reorder_threshold int not null +); + +create table public.inventory ( + id bigint generated always as identity primary key, + warehouse_id bigint not null references public.warehouses (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity >= 0) +); + +alter table public.warehouses enable row level security; +alter table public.suppliers enable row level security; +alter table public.products enable row level security; +alter table public.inventory enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.warehouses, public.suppliers, public.products, + public.inventory to service_role; + +-- Seed data. +insert into public.warehouses (name) values + ('North DC'), + ('South DC'), + ('West DC'); + +insert into public.suppliers (name, email) values + ('Acme Supplies', 'acme@example.com'), + ('Global Parts', 'parts@example.com'); + +insert into public.products (name, supplier_id, reorder_threshold) values + ('Widget', 1, 20), + ('Gadget', 2, 15), + ('Gizmo', 1, 10); + +-- warehouse_id, product_id, quantity. Below-threshold rows are the alerts; +-- West DC's Widget sits exactly at threshold and must NOT alert (strict +-- less-than), and West DC has no Gizmo row at all (untracked combos are +-- simply absent, not a zero-quantity alert). +insert into public.inventory (warehouse_id, product_id, quantity) values + (1, 1, 5), -- North DC / Widget: 5 < 20 -> alert + (1, 2, 30), -- North DC / Gadget: 30 >= 15 -> ok + (1, 3, 3), -- North DC / Gizmo: 3 < 10 -> alert + (2, 1, 25), -- South DC / Widget: 25 >= 20 -> ok + (2, 2, 2), -- South DC / Gadget: 2 < 15 -> alert + (2, 3, 12), -- South DC / Gizmo: 12 >= 10 -> ok + (3, 1, 20), -- West DC / Widget: 20 >= 20 -> ok (boundary, not strictly below) + (3, 2, 0); -- West DC / Gadget: 0 < 15 -> alert diff --git a/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts new file mode 100644 index 00000000..7921bd26 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts @@ -0,0 +1,316 @@ +import { + type CheckResult, + type CommandResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Legacy → new API key migration (regression): the seeded app authenticates +// with the local stack's legacy demo JWTs (anon + service_role) and works +// out of the box; the task is to move it to the new sb_publishable_… / +// sb_secret_… keys without breaking it. Scored on behavior (both scripts +// still print the right data — the stats script's RLS-bypassing count only +// works with a genuine secret key), on the legacy JWTs being gone, and on +// the key boundary staying intact (the public script must not end up holding +// the secret key). + +const APP_DIR = 'app'; + +// Header+payload prefix shared by both legacy local demo JWTs +// ({"iss":"supabase-demo",…} signed with the default local JWT secret) — +// matching on it catches either key regardless of the signature bytes. +const LEGACY_JWT_MARKER = 'eyJpc3MiOiJzdXBhYmFzZS1kZW1vIi'; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const publishableKey = str(status.PUBLISHABLE_KEY); + const secretKey = str(status.SECRET_KEY); + if (!publishableKey || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the keys, + // not npm. A no-op when the agent already installed dependencies. + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, + { timeoutMs: 180_000 } + ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } + + // Ground truth from the seeded database. + const { rows: publishedRows } = await ctx.query( + `select title from public.posts where published order by title asc` + ); + const expectedTitles = publishedRows.map((row) => row.title); + const { rows: draftRows } = await ctx.query( + `select count(*)::int as n from public.posts where not published` + ); + const expectedDrafts = Number(draftRows[0]?.n ?? -1); + + // 1. The public script still lists published posts (client-key path). + const posts = await ctx.exec(`cd ${APP_DIR} && npm run -s posts`, { + timeoutMs: 60_000, + }); + const postTitles = parseJson(posts.stdout, '[', ']'); + checks.push({ + name: 'posts script still lists published posts', + passed: + JSON.stringify(postTitles ?? null) === JSON.stringify(expectedTitles), + notes: postTitles + ? `got ${JSON.stringify(postTitles)}` + : `no JSON output — ${preview(posts.stderr || posts.stdout)}`, + }); + + // 2. The internal script still counts drafts. Drafts are invisible to the + // publishable key (RLS), so a correct count proves a working secret key. + const stats = await ctx.exec(`cd ${APP_DIR} && npm run -s stats`, { + timeoutMs: 60_000, + }); + const statsOut = parseJson(stats.stdout, '{', '}') as + | { drafts?: unknown } + | undefined; + checks.push({ + name: 'stats script still counts drafts (secret key bypasses RLS)', + passed: statsOut?.drafts === expectedDrafts, + notes: statsOut + ? `got ${JSON.stringify(statsOut)}, expected ${expectedDrafts} drafts` + : `no JSON output — ${preview(stats.stderr || stats.stdout)}`, + }); + + // 3. The legacy JWTs are gone from the app (env files included). + const legacyScan = await ctx.exec( + `grep -rl --exclude-dir=node_modules '${LEGACY_JWT_MARKER}' ${APP_DIR} || true` + ); + checks.push({ + name: 'legacy anon/service_role JWTs removed from the app', + passed: legacyScan.stdout.trim() === '', + notes: + legacyScan.stdout.trim().replace(/\s+/g, ', ') || + 'no legacy JWTs found', + }); + + // 4. Key boundary intact: the public script must not hold the secret key, + // neither as a literal nor via an env var that resolves to it. + checks.push(await publicScriptKeyCheck(ctx)); + + // 5. Prove both scripts actually depend on the keys, not just that their + // output happens to match: corrupt the live key value on disk and + // confirm the same script now breaks. Without this, a stub that already + // knows the expected titles/count — with no real Supabase call at all — + // would pass checks 1-4 for free. + checks.push( + await keyDependencyCheck(ctx, 'posts', publishableKey, 'publishable key') + ); + checks.push( + await keyDependencyCheck(ctx, 'stats', secretKey, 'secret key') + ); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +/** + * The public (posts) script must not use the secret key: no sb_secret_ + * literal in its source, and no reference to an env var whose value in .env + * is a secret key. + */ +async function publicScriptKeyCheck( + ctx: LocalStackEvalContext +): Promise { + const NAME = 'public script does not hold the secret key'; + const entry = await resolveScriptEntry(ctx, 'posts'); + const source = await ctx + .readFile(`${APP_DIR}/${entry}`) + .catch(() => undefined); + if (source === undefined) { + return { + name: NAME, + passed: false, + notes: `could not read ${APP_DIR}/${entry} to inspect`, + }; + } + if (source.includes('sb_secret_')) { + return { + name: NAME, + passed: false, + notes: `${entry} contains an sb_secret_ literal`, + }; + } + const env = await ctx.readFile(`${APP_DIR}/.env`).catch(() => ''); + const secretVars = parseEnv(env) + .filter(([, value]) => value.startsWith('sb_secret_')) + .map(([key]) => key); + const leaked = secretVars.filter((name) => source.includes(name)); + return { + name: NAME, + passed: leaked.length === 0, + notes: leaked.length + ? `${entry} references secret-key env var(s): ${leaked.join(', ')}` + : `${entry} holds no secret-key reference`, + }; +} + +/** + * Corrupts the on-disk copy of `keyValue` (in `.env`, or in the script's + * source if it's hardcoded there instead) and re-runs the given npm script, + * expecting it to break — proving the script reads the key at call time + * rather than being hardcoded or key-agnostic. Restores the original + * content afterward either way. + */ +async function keyDependencyCheck( + ctx: LocalStackEvalContext, + script: 'posts' | 'stats', + keyValue: string, + label: string +): Promise { + const NAME = `${script} script actually depends on its ${label}`; + const envPath = `${APP_DIR}/.env`; + const entry = await resolveScriptEntry(ctx, script); + const entryPath = `${APP_DIR}/${entry}`; + + const env = await ctx.readFile(envPath).catch(() => undefined); + const targetPath = env?.includes(keyValue) + ? envPath + : await ctx + .readFile(entryPath) + .then((source) => (source.includes(keyValue) ? entryPath : undefined)) + .catch(() => undefined); + if (targetPath === undefined) { + return { + name: NAME, + passed: false, + notes: `could not find the live ${label} on disk (checked ${envPath} and ${entryPath}) to corrupt`, + }; + } + + const original = await ctx.readFile(targetPath); + const corrupted = original.replaceAll( + keyValue, + `${keyValue}-corrupted-by-eval` + ); + await writeFile(ctx, targetPath, corrupted); + const run = await ctx + .exec(`cd ${APP_DIR} && npm run -s ${script}`, { timeoutMs: 60_000 }) + .finally(() => writeFile(ctx, targetPath, original)); + + const stillLooksValid = + script === 'posts' + ? parseJson(run.stdout, '[', ']') !== undefined + : (parseJson(run.stdout, '{', '}') as { drafts?: unknown } | undefined) + ?.drafts !== undefined; + return { + name: NAME, + passed: !run.ok || !stillLooksValid, + notes: + !run.ok || !stillLooksValid + ? `broke as expected with a corrupted ${label}` + : `still produced ${preview(run.stdout)} with a corrupted ${label} — looks hardcoded or key-agnostic`, + }; +} + +function writeFile( + ctx: LocalStackEvalContext, + path: string, + content: string +): Promise { + const encoded = Buffer.from(content, 'utf-8').toString('base64'); + return ctx.exec(`echo ${encoded} | base64 -d > ${path}`); +} + +/** File the given npm script runs, e.g. `posts` → `posts.mjs`. */ +async function resolveScriptEntry( + ctx: LocalStackEvalContext, + script: string +): Promise { + const fallback = `${script}.mjs`; + try { + const pkg = JSON.parse(await ctx.readFile(`${APP_DIR}/package.json`)) as { + scripts?: Record; + }; + const command = pkg.scripts?.[script] ?? ''; + return command.match(/[\w./-]+\.(?:mjs|cjs|js|ts)/)?.[0] ?? fallback; + } catch { + return fallback; + } +} + +function parseEnv(content: string): Array<[string, string]> { + return content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .flatMap((line) => { + const eq = line.indexOf('='); + if (eq === -1) return []; + const value = line + .slice(eq + 1) + .trim() + .replace(/^['"]|['"]$/g, ''); + return [[line.slice(0, eq).trim(), value] as [string, string]]; + }); +} + +function parseJson( + stdout: string, + open: string, + close: string +): unknown | undefined { + const start = stdout.indexOf(open); + const end = stdout.lastIndexOf(close); + if (start === -1 || end <= start) return undefined; + try { + return JSON.parse(stdout.slice(start, end + 1)); + } catch { + return undefined; + } +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md new file mode 100644 index 00000000..7c59bf8c --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md @@ -0,0 +1,27 @@ +--- +stage: resolve +suite: regression +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - auth +topic: + - sdk + - security +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: https://github.com/orgs/supabase/discussions/29260 +--- + +Heads-up from the platform team: the legacy JWT-based API keys (`anon` / +`service_role`) are going away for our projects soon, in favor of the new +publishable/secret keys. The little blog tooling app in `app/` still uses the +legacy keys. + +Migrate it over. Both scripts need to keep working — `npm run posts` and +`npm run stats` (run them from `app/`). The local Supabase project in +`supabase/` is already running. diff --git a/evals/resolve-sdk-001-legacy-key-migration/README.md b/evals/resolve-sdk-001-legacy-key-migration/README.md new file mode 100644 index 00000000..c5bc32d5 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/README.md @@ -0,0 +1,24 @@ +# resolve-sdk-001-legacy-key-migration + +A working blog-tooling app (`local/app/`) authenticates with the **legacy +local demo JWTs** — the deterministic `anon` / `service_role` keys every +local stack issues when `supabase/config.toml` doesn't override the JWT +secret (payload `{"iss":"supabase-demo",…}` signed with the default +`super-secret-jwt-token-with-at-least-32-characters-long`). That's what lets +the seed hardcode valid keys in `.env` before the stack exists. + +The task is to migrate the app to the new `sb_publishable_…` / `sb_secret_…` +keys (see the motivation link). The scorer checks behavior, not process: + +1. `npm run posts` still prints the published titles (client-key path). +2. `npm run stats` still prints the draft count — drafts are hidden from the + publishable key by RLS, so a correct count proves a real secret key. +3. No legacy demo JWT remains anywhere in `app/` (matched on the shared + header+payload prefix, so it catches both keys). +4. The public script doesn't end up holding the secret key, directly or via + an env var that resolves to one. + +Assumptions to keep in mind: the pinned `cliVersion` must expose +`PUBLISHABLE_KEY` / `SECRET_KEY` in `supabase status -o json`, and the stack +must still accept legacy JWTs by default (true as of 2.109.1) or the seeded +app would be broken before the agent starts. diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/.env b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env new file mode 100644 index 00000000..e9c51c59 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env @@ -0,0 +1,4 @@ +# Supabase keys for local dev (from `supabase status`) +SUPABASE_URL=http://127.0.0.1:54321 +SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0 +SUPABASE_SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json new file mode 100644 index 00000000..5cab9c72 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json @@ -0,0 +1,12 @@ +{ + "name": "blog-tools", + "private": true, + "type": "module", + "scripts": { + "posts": "node --env-file=.env posts.mjs", + "stats": "node --env-file=.env stats.mjs" + }, + "dependencies": { + "@supabase/supabase-js": "^2.58.0" + } +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs new file mode 100644 index 00000000..749820f0 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Public site: lists published post titles. Runs with the project's public +// (client-side) API key, so RLS applies. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_ANON_KEY +); + +const { data, error } = await supabase + .from('posts') + .select('title') + .order('title'); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify(data.map((post) => post.title))); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs new file mode 100644 index 00000000..a7155ccd --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Internal tooling: counts unpublished drafts across all posts. Trusted +// backend only — runs with the project's server-side key, which bypasses RLS. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_SERVICE_ROLE_KEY +); + +const { count, error } = await supabase + .from('posts') + .select('*', { count: 'exact', head: true }) + .eq('published', false); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify({ drafts: count })); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml new file mode 100644 index 00000000..d9722d7e --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-key-migration" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql new file mode 100644 index 00000000..28e2c5b8 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql @@ -0,0 +1,27 @@ +-- Blog posts. Published posts are public (readable with the client key); +-- drafts are only reachable by trusted backend code that bypasses RLS. +create table public.posts ( + id bigint generated always as identity primary key, + title text not null, + published boolean not null default false +); + +alter table public.posts enable row level security; + +create policy "anyone can read published posts" + on public.posts + for select + to anon, authenticated + using (published); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows for client roles; +-- service_role (trusted backend) bypasses RLS but still needs the privilege. +grant select on public.posts to anon, authenticated, service_role; + +insert into public.posts (title, published) values + ('Announcing vector buckets', true), + ('Realtime broadcast tips', true), + ('Row level security explained', true), + ('DRAFT: pricing update', false), + ('DRAFT: roadmap 2027', false);