From 1953c05d3abc33878426b95b78fae333923f3853 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:50:20 -0400 Subject: [PATCH 01/10] feat: promote build-rls-003-org-roles-permissions to benchmark suite --- .../EVAL.ts | 65 ++++++++++++++----- .../PROMPT.md | 30 +-------- .../remote/project.sql | 6 ++ 3 files changed, 58 insertions(+), 43 deletions(-) diff --git a/evals/build-rls-003-org-roles-permissions/EVAL.ts b/evals/build-rls-003-org-roles-permissions/EVAL.ts index ea7ee12b..97c93d74 100644 --- a/evals/build-rls-003-org-roles-permissions/EVAL.ts +++ b/evals/build-rls-003-org-roles-permissions/EVAL.ts @@ -1,4 +1,5 @@ import type { CheckResult, ToolScorer } from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; const ORG_A = '11111111-1111-1111-1111-111111111111'; const ORG_B = '22222222-2222-2222-2222-222222222222'; @@ -120,24 +121,56 @@ RETURNING id; passed: editorUpdatesAdmin.length === 0, }); - await q( - asUser( - ADMIN_A, - ` -DELETE FROM documents -WHERE id = '10000000-0000-0000-0000-000000000001'; - ` - ) - ); - const { rows: adminSoftDelete } = await q( - `SELECT id, deleted_at FROM documents WHERE id = '10000000-0000-0000-0000-000000000001';` - ); - checks.push({ - name: 'admin delete soft-deletes document in org', - passed: + // A too-strict WITH CHECK is a known RLS soft-delete footgun. Treat it + // as a failed check instead of crashing the run. + let adminSoftDeletePassed = false; + try { + const { rows: adminSoftDelete } = await q( + asUser( + ADMIN_A, + stripIndent` + UPDATE documents + SET deleted_at = now() + WHERE id = '10000000-0000-0000-0000-000000000001' + RETURNING id, deleted_at; + ` + ) + ); + adminSoftDeletePassed = adminSoftDelete.length === 1 && adminSoftDelete[0]?.id === '10000000-0000-0000-0000-000000000001' && - Boolean(adminSoftDelete[0]?.deleted_at), + Boolean(adminSoftDelete[0]?.deleted_at); + } catch { + await resetTx(); + } + checks.push({ + name: 'admin can mark a document deleted in their org', + passed: adminSoftDeletePassed, + }); + + // Agents either block hard deletes via grants/RLS or intercept DELETE + // with a trigger that soft-deletes instead. Accept either as long as + // the row is never actually removed. + let hardDeletePrevented = false; + try { + await q( + asUser( + ADMIN_A, + `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000001';`, + 'ROLLBACK' + ) + ); + const { rows: stillThere } = await q( + `SELECT id FROM documents WHERE id = '10000000-0000-0000-0000-000000000001';` + ); + hardDeletePrevented = stillThere.length === 1; + } catch { + hardDeletePrevented = true; + await resetTx(); + } + checks.push({ + name: 'documents are never hard-deleted', + passed: hardDeletePrevented, }); const { rows: adminCrossOrg } = await q( diff --git a/evals/build-rls-003-org-roles-permissions/PROMPT.md b/evals/build-rls-003-org-roles-permissions/PROMPT.md index 75256878..002a1381 100644 --- a/evals/build-rls-003-org-roles-permissions/PROMPT.md +++ b/evals/build-rls-003-org-roles-permissions/PROMPT.md @@ -1,6 +1,6 @@ --- stage: build -suite: other +suite: benchmark interface: mcp product: - database @@ -8,31 +8,7 @@ product: topic: - rls - security +motivation: AI-1006, FDBKIN-19292, FDBKIN-8277, FDBKIN-9175 --- -You are working on a Supabase project for a multi-tenant document app. - -The schema has three tables already created and seeded: - -```sql --- memberships -user_id uuid, org_id uuid, role text - --- documents -id uuid, org_id uuid, owner_id uuid, title text, body text, deleted_at timestamptz - --- document_audit -id uuid, document_id uuid, actor_id uuid, action text, ts timestamptz -``` - -Add the RLS policies and database logic needed for authenticated users: - -1. Viewers can read active documents in orgs where they are members. -2. Editors can read active documents in their orgs, insert documents they own in their orgs, and update or delete only documents they own. -3. Admins can read, update, and delete any active document in orgs where they are admins. -4. Soft-deleted documents (`deleted_at IS NOT NULL`) should not be visible through normal reads. -5. Deletes should be soft deletes by setting `deleted_at`; do not hard-delete rows. -6. Every insert, update, and soft-delete should write a row to `document_audit` with the acting user. - -Apply the required database changes. End your turn when you believe the -policies and audit behavior are in place. +Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what. diff --git a/evals/build-rls-003-org-roles-permissions/remote/project.sql b/evals/build-rls-003-org-roles-permissions/remote/project.sql index 3a3c9625..9ea109cb 100644 --- a/evals/build-rls-003-org-roles-permissions/remote/project.sql +++ b/evals/build-rls-003-org-roles-permissions/remote/project.sql @@ -26,6 +26,12 @@ GRANT SELECT ON memberships TO authenticated; GRANT SELECT, INSERT, UPDATE, DELETE ON documents TO authenticated; GRANT SELECT, INSERT ON document_audit TO authenticated; +-- Mimics someone accepting Studio's default read-access policy template +-- without customizing it. Permissive policies OR together in Postgres RLS, +-- so this stays a live hole unless the agent notices and drops it. +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; +CREATE POLICY "Enable read access for all users" ON documents FOR SELECT USING (true); + INSERT INTO memberships (user_id, org_id, role) VALUES ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', '11111111-1111-1111-1111-111111111111', 'admin'), ('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', '11111111-1111-1111-1111-111111111111', 'editor'), From fc48e9f9eca9760cbd5aeb71cb4e87c713a63f8c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:03:50 +0000 Subject: [PATCH 02/10] chore: refresh eval results --- apps/web/src/data/eval-results.json | 1114 ++++++++++++++++++++++++++- 1 file changed, 1076 insertions(+), 38 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 5b28a57e..cd410300 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -660,6 +660,92 @@ "attempts": 1, "sourcePath": "claude-code-opus-5/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/build-rls-003-org-roles-permissions.json" + }, { "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", @@ -2705,6 +2791,86 @@ "attempts": 2, "sourcePath": "claude-code-opus-5-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "claude-code-opus-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/build-rls-003-org-roles-permissions.json" + }, { "experiment": "claude-code-opus-5-no-skills", "experimentSuite": "no-skills", @@ -4718,6 +4884,92 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-rls-003-org-roles-permissions.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -6080,57 +6332,63 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-storage-001-private-bucket-access", + "eval": "build-rls-003-org-roles-permissions", "stage": "build", "product": [ - "storage", - "database" + "database", + "auth" ], "topic": [ "rls", - "sdk" + "security" ], "suite": "benchmark", "interface": "mcp", "passed": true, "checks": [ { - "name": "bucket user-files exists", + "name": "RLS enabled on documents", "passed": true }, { - "name": "bucket user-files is private", + "name": "viewer sees active org documents only", "passed": true }, { - "name": "RLS still enabled on storage.objects", + "name": "viewer cannot insert", "passed": true }, { - "name": "user A lists only own files", - "passed": true, - "notes": "saw: 019fb0e7-0613-730a-ae13-0efd0c195137/receipt-alpha.pdf, 019fb0e7-0613-730a-ae13-0efd0c195137/receipt-beta.pdf" + "name": "editor can insert own org document", + "passed": true }, { - "name": "user B cannot read user A files", + "name": "editor can update own document", "passed": true }, { - "name": "anon reads no files", + "name": "editor cannot update another user's document", "passed": true }, { - "name": "user A can upload into own folder", + "name": "admin can mark a document deleted in their org", "passed": true }, { - "name": "user B cannot upload into user A folder", + "name": "documents are never hard-deleted", "passed": true }, { - "name": "configured private per-user storage access", - "passed": true, - "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK on storage.objects, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for sharing." + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true } ], "skills": { @@ -6140,10 +6398,10 @@ "docs": { "calls": [] }, - "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", - "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" + "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-003-org-roles-permissions.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -6154,29 +6412,103 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-tests-001-rls-tenant-isolation", + "eval": "build-storage-001-private-bucket-access", "stage": "build", "product": [ + "storage", "database" ], "topic": [ - "tests", - "rls" + "rls", + "sdk" ], "suite": "benchmark", - "interface": "cli", + "interface": "mcp", "passed": true, "checks": [ { - "name": "pgTAP test file(s) written under supabase/tests/", - "passed": true, - "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" + "name": "bucket user-files exists", + "passed": true }, { - "name": "pgTAP isolation tests ran and pass", - "passed": true, - "notes": "6 passed, 2 failed" - }, + "name": "bucket user-files is private", + "passed": true + }, + { + "name": "RLS still enabled on storage.objects", + "passed": true + }, + { + "name": "user A lists only own files", + "passed": true, + "notes": "saw: 019fb0e7-0613-730a-ae13-0efd0c195137/receipt-alpha.pdf, 019fb0e7-0613-730a-ae13-0efd0c195137/receipt-beta.pdf" + }, + { + "name": "user B cannot read user A files", + "passed": true + }, + { + "name": "anon reads no files", + "passed": true + }, + { + "name": "user A can upload into own folder", + "passed": true + }, + { + "name": "user B cannot upload into user A folder", + "passed": true + }, + { + "name": "configured private per-user storage access", + "passed": true, + "judgeNotes": "Meets requirements: private user-files bucket, authenticated owner-scoped SELECT and INSERT policies with WITH CHECK on storage.objects, no RLS disabling or public access, and supabase-js createSignedUrl with expiry for sharing." + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-storage-001-private-bucket-access.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-tests-001-rls-tenant-isolation", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "tests", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "pgTAP test file(s) written under supabase/tests/", + "passed": true, + "notes": "1 file(s): supabase/tests/database/tenant_isolation_test.sql" + }, + { + "name": "pgTAP isolation tests ran and pass", + "passed": true, + "notes": "6 passed, 2 failed" + }, { "name": "agent correctly identifies the posts isolation bug from test results", "passed": true, @@ -7563,6 +7895,121 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"row level security policies update select with check soft delete audit trigger\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/storage/security/access-control", + "title": "Storage Access Control" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 77496 + } + ] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-rls-003-org-roles-permissions.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -9915,6 +10362,199 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security policies security definer trigger soft delete audit log\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#access-token", + "title": "Access token" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#authentication", + "title": "Authentication" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#authenticator-app", + "title": "Authenticator app" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#authorization", + "title": "Authorization" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#identity-provider", + "title": "Identity provider" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#json-web-token-jwt", + "title": "JSON Web Token (JWT)" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#jwt-signing-secret", + "title": "JWT signing secret" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#multi-factor-authentication-mfa-or-2fa", + "title": "Multi-factor authentication (MFA or 2FA)" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#nonce", + "title": "Nonce" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#oauth", + "title": "OAuth" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#oidc", + "title": "OIDC" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#one-time-password-otp", + "title": "One-time password (OTP)" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#password-hashing-function", + "title": "Password hashing function" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#password-strength", + "title": "Password strength" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#pkce", + "title": "PKCE" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#provider-refresh-token", + "title": "Provider refresh token" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#provider-tokens", + "title": "Provider tokens" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token", + "title": "Refresh token" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token-flow", + "title": "Refresh token flow" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#replay-attack", + "title": "Replay attack" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#row-level-security-policies-rls", + "title": "Row level security policies (RLS)" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#saml", + "title": "SAML" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#session", + "title": "Session" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#single-sign-on-sso", + "title": "Single-sign on (SSO)" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary#time-based-one-time-password-totp", + "title": "Time-based one-time password (TOTP)" + } + ], + "resultChars": 20326 + } + ] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -12208,18 +12848,131 @@ "title": "JWT Claims Reference" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" + } + ], + "resultChars": 114347 + } + ] + }, + "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", + "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking change RLS policies triggers audit soft delete", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security organization membership roles soft delete audit trigger security definer\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" } ], - "resultChars": 114347 + "resultChars": 27158 } ] }, - "prompt": "Build and serve a Supabase Edge Function named `user-stats` for this project,\nreachable over HTTP at `/functions/v1/user-stats`.\n\nOur product stores per-user metrics in the existing `user_stats` table.\n\nTwo very different callers need to hit this one endpoint:\n\n1. **Our mobile app**, acting for a signed-in user. It sends that user's\n Supabase access token. The endpoint should return the user's own stats.\n\n2. **Our internal billing service**, a trusted backend with no signed-in user.\n It authenticates with the project's secret (service-role) key in the `apikey`\n header, and names the target user with a `user_id` in the JSON request body.\n It needs that user's stats.\n\nReturn the matching rows as JSON. The endpoint must be secure: only ever serve\nstats to a caller that is genuinely entitled to them, and turn away callers that\nare not.", - "promptSourcePath": "evals/build-functions-005-dual-auth-user-secret/PROMPT.md", - "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-functions-005-dual-auth-user-secret.json" + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions.json" }, { "experiment": "codex-gpt-5.6", @@ -14730,6 +15483,115 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres row level security auth.uid security definer policies triggers audit soft delete\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/architecture", + "title": "Auth architecture" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/triggers", + "title": "Postgres Triggers" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + } + ], + "resultChars": 56692 + } + ] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-rls-003-org-roles-permissions.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", @@ -16406,6 +17268,103 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 93795 + } + ] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-rls-003-org-roles-permissions.json" + }, { "experiment": "opencode-kimi-k3", "experimentSuite": "benchmark", @@ -17974,6 +18933,85 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-rls-003-org-roles-permissions", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on documents", + "passed": true + }, + { + "name": "viewer sees active org documents only", + "passed": true + }, + { + "name": "viewer cannot insert", + "passed": true + }, + { + "name": "editor can insert own org document", + "passed": true + }, + { + "name": "editor can update own document", + "passed": true + }, + { + "name": "editor cannot update another user's document", + "passed": true + }, + { + "name": "admin can mark a document deleted in their org", + "passed": true + }, + { + "name": "documents are never hard-deleted", + "passed": true + }, + { + "name": "admin cannot affect another org", + "passed": true + }, + { + "name": "WITH CHECK blocks editor from moving document to another org", + "passed": true + }, + { + "name": "write creates audit row with acting user", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-rls-003-org-roles-permissions.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", From 21c5db30a2e460754c4ab9ae5897c9587dbfa810 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 4 Aug 2026 09:56:07 -0400 Subject: [PATCH 03/10] refactor: scope build-rls-003 down to org/role RLS only Drops soft-delete and audit-log requirements. They weren't part of the actual motivation for promoting this scenario (org/role RLS complexity), and every scorer bug we hit lived in that bolted-on logic. --- .../EVAL.ts | 175 +++++++----------- .../PROMPT.md | 4 +- .../remote/project.sql | 21 +-- 3 files changed, 79 insertions(+), 121 deletions(-) diff --git a/evals/build-rls-003-org-roles-permissions/EVAL.ts b/evals/build-rls-003-org-roles-permissions/EVAL.ts index 97c93d74..ff865d69 100644 --- a/evals/build-rls-003-org-roles-permissions/EVAL.ts +++ b/evals/build-rls-003-org-roles-permissions/EVAL.ts @@ -11,13 +11,13 @@ const asUser = ( sub: string, body: string, finish: 'COMMIT' | 'ROLLBACK' = 'COMMIT' -) => ` -BEGIN; -SET LOCAL ROLE authenticated; -SET LOCAL request.jwt.claim.sub = '${sub}'; -SET LOCAL request.jwt.claim.role = 'authenticated'; -${body} -${finish}; +) => stripIndent` + BEGIN; + SET LOCAL ROLE authenticated; + SET LOCAL request.jwt.claim.sub = '${sub}'; + SET LOCAL request.jwt.claim.role = 'authenticated'; + ${body} + ${finish}; `; const scorer: ToolScorer = async (ctx) => { @@ -34,20 +34,18 @@ const scorer: ToolScorer = async (ctx) => { try { const { rows: rls } = await q( - `SELECT relname, relrowsecurity FROM pg_class WHERE relname IN ('documents', 'document_audit');` + `SELECT relrowsecurity FROM pg_class WHERE relname = 'documents';` ); checks.push({ name: 'RLS enabled on documents', - passed: rls.some( - (row) => row.relname === 'documents' && row.relrowsecurity === true - ), + passed: rls[0]?.relrowsecurity === true, }); const { rows: viewerReads } = await q( asUser(VIEWER_A, `SELECT title FROM documents ORDER BY title;`) ); checks.push({ - name: 'viewer sees active org documents only', + name: 'viewer sees only documents in their org', passed: viewerReads.length === 2 && viewerReads.map((row) => row.title).join(',') === @@ -59,9 +57,9 @@ const scorer: ToolScorer = async (ctx) => { await q( asUser( VIEWER_A, - ` -INSERT INTO documents (org_id, owner_id, title, body) -VALUES ('${ORG_A}', '${VIEWER_A}', 'viewer insert', 'should fail'); + stripIndent` + INSERT INTO documents (org_id, owner_id, title, body) + VALUES ('${ORG_A}', '${VIEWER_A}', 'viewer insert', 'should fail'); ` ) ); @@ -74,10 +72,10 @@ VALUES ('${ORG_A}', '${VIEWER_A}', 'viewer insert', 'should fail'); const { rows: editorInsert } = await q( asUser( EDITOR_A, - ` -INSERT INTO documents (org_id, owner_id, title, body) -VALUES ('${ORG_A}', '${EDITOR_A}', 'editor insert', 'allowed') -RETURNING id; + stripIndent` + INSERT INTO documents (org_id, owner_id, title, body) + VALUES ('${ORG_A}', '${EDITOR_A}', 'editor insert', 'allowed') + RETURNING id; `, 'ROLLBACK' ) @@ -90,11 +88,11 @@ RETURNING id; const { rows: editorOwnUpdate } = await q( asUser( EDITOR_A, - ` -UPDATE documents -SET body = 'editor changed own document' -WHERE id = '10000000-0000-0000-0000-000000000002' -RETURNING id; + stripIndent` + UPDATE documents + SET body = 'editor changed own document' + WHERE id = '10000000-0000-0000-0000-000000000002' + RETURNING id; `, 'ROLLBACK' ) @@ -107,11 +105,11 @@ RETURNING id; const { rows: editorUpdatesAdmin } = await q( asUser( EDITOR_A, - ` -UPDATE documents -SET body = 'editor changed admin document' -WHERE id = '10000000-0000-0000-0000-000000000001' -RETURNING id; + stripIndent` + UPDATE documents + SET body = 'editor changed admin document' + WHERE id = '10000000-0000-0000-0000-000000000001' + RETURNING id; `, 'ROLLBACK' ) @@ -121,66 +119,62 @@ RETURNING id; passed: editorUpdatesAdmin.length === 0, }); - // A too-strict WITH CHECK is a known RLS soft-delete footgun. Treat it - // as a failed check instead of crashing the run. - let adminSoftDeletePassed = false; + let editorDeleteBlocked = false; try { - const { rows: adminSoftDelete } = await q( + const { rows } = await q( asUser( - ADMIN_A, - stripIndent` - UPDATE documents - SET deleted_at = now() - WHERE id = '10000000-0000-0000-0000-000000000001' - RETURNING id, deleted_at; - ` + EDITOR_A, + `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000001' RETURNING id;`, + 'ROLLBACK' ) ); - adminSoftDeletePassed = - adminSoftDelete.length === 1 && - adminSoftDelete[0]?.id === '10000000-0000-0000-0000-000000000001' && - Boolean(adminSoftDelete[0]?.deleted_at); + editorDeleteBlocked = rows.length === 0; } catch { + editorDeleteBlocked = true; await resetTx(); } checks.push({ - name: 'admin can mark a document deleted in their org', - passed: adminSoftDeletePassed, + name: "editor cannot delete another user's document", + passed: editorDeleteBlocked, }); - // Agents either block hard deletes via grants/RLS or intercept DELETE - // with a trigger that soft-deletes instead. Accept either as long as - // the row is never actually removed. - let hardDeletePrevented = false; - try { - await q( - asUser( - ADMIN_A, - `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000001';`, - 'ROLLBACK' - ) - ); - const { rows: stillThere } = await q( - `SELECT id FROM documents WHERE id = '10000000-0000-0000-0000-000000000001';` - ); - hardDeletePrevented = stillThere.length === 1; - } catch { - hardDeletePrevented = true; - await resetTx(); - } + const { rows: adminUpdate } = await q( + asUser( + ADMIN_A, + stripIndent` + UPDATE documents + SET body = 'admin changed editor document' + WHERE id = '10000000-0000-0000-0000-000000000002' + RETURNING id; + `, + 'ROLLBACK' + ) + ); checks.push({ - name: 'documents are never hard-deleted', - passed: hardDeletePrevented, + name: 'admin can update any document in their org', + passed: adminUpdate.length === 1, + }); + + const { rows: adminDelete } = await q( + asUser( + ADMIN_A, + `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000002' RETURNING id;`, + 'ROLLBACK' + ) + ); + checks.push({ + name: 'admin can delete any document in their org', + passed: adminDelete.length === 1, }); const { rows: adminCrossOrg } = await q( asUser( ADMIN_A, - ` -UPDATE documents -SET deleted_at = now() -WHERE id = '20000000-0000-0000-0000-000000000001' -RETURNING id; + stripIndent` + UPDATE documents + SET body = 'admin touched another org' + WHERE id = '20000000-0000-0000-0000-000000000001' + RETURNING id; `, 'ROLLBACK' ) @@ -195,11 +189,11 @@ RETURNING id; const { rows } = await q( asUser( EDITOR_A, - ` -UPDATE documents -SET org_id = '${ORG_B}' -WHERE id = '10000000-0000-0000-0000-000000000002' -RETURNING id; + stripIndent` + UPDATE documents + SET org_id = '${ORG_B}' + WHERE id = '10000000-0000-0000-0000-000000000002' + RETURNING id; `, 'ROLLBACK' ) @@ -213,31 +207,6 @@ RETURNING id; name: 'WITH CHECK blocks editor from moving document to another org', passed: orgReassignmentBlocked, }); - - const { rows: auditedUpdate } = await q( - asUser( - EDITOR_A, - ` -UPDATE documents -SET body = 'audit verifier' -WHERE id = '10000000-0000-0000-0000-000000000002' -RETURNING id; - ` - ) - ); - const { rows: auditRows } = await q( - `SELECT actor_id, document_id FROM document_audit WHERE document_id = '10000000-0000-0000-0000-000000000002';` - ); - checks.push({ - name: 'write creates audit row with acting user', - passed: - auditedUpdate.length === 1 && - auditRows.some( - (row) => - row.actor_id === EDITOR_A && - row.document_id === '10000000-0000-0000-0000-000000000002' - ), - }); } catch (error) { const msg = error instanceof Error ? error.message : String(error); checks.push({ diff --git a/evals/build-rls-003-org-roles-permissions/PROMPT.md b/evals/build-rls-003-org-roles-permissions/PROMPT.md index 002a1381..e80cc53c 100644 --- a/evals/build-rls-003-org-roles-permissions/PROMPT.md +++ b/evals/build-rls-003-org-roles-permissions/PROMPT.md @@ -8,7 +8,7 @@ product: topic: - rls - security -motivation: AI-1006, FDBKIN-19292, FDBKIN-8277, FDBKIN-9175 +motivation: AI-1006, FDBKIN-19292 --- -Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what. +Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. diff --git a/evals/build-rls-003-org-roles-permissions/remote/project.sql b/evals/build-rls-003-org-roles-permissions/remote/project.sql index 9ea109cb..efdcb524 100644 --- a/evals/build-rls-003-org-roles-permissions/remote/project.sql +++ b/evals/build-rls-003-org-roles-permissions/remote/project.sql @@ -10,21 +10,11 @@ CREATE TABLE documents ( org_id uuid NOT NULL, owner_id uuid NOT NULL, title text NOT NULL, - body text NOT NULL, - deleted_at timestamptz -); - -CREATE TABLE document_audit ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), - document_id uuid NOT NULL, - actor_id uuid NOT NULL, - action text NOT NULL, - ts timestamptz NOT NULL DEFAULT now() + body text NOT NULL ); GRANT SELECT ON memberships TO authenticated; GRANT SELECT, INSERT, UPDATE, DELETE ON documents TO authenticated; -GRANT SELECT, INSERT ON document_audit TO authenticated; -- Mimics someone accepting Studio's default read-access policy template -- without customizing it. Permissive policies OR together in Postgres RLS, @@ -38,8 +28,7 @@ INSERT INTO memberships (user_id, org_id, role) VALUES ('cccccccc-cccc-cccc-cccc-cccccccccccc', '11111111-1111-1111-1111-111111111111', 'viewer'), ('dddddddd-dddd-dddd-dddd-dddddddddddd', '22222222-2222-2222-2222-222222222222', 'editor'); -INSERT INTO documents (id, org_id, owner_id, title, body, deleted_at) VALUES - ('10000000-0000-0000-0000-000000000001', '11111111-1111-1111-1111-111111111111', 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'Admin plan', 'org A admin document', NULL), - ('10000000-0000-0000-0000-000000000002', '11111111-1111-1111-1111-111111111111', 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'Editor draft', 'org A editor document', NULL), - ('10000000-0000-0000-0000-000000000003', '11111111-1111-1111-1111-111111111111', 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'Deleted draft', 'org A deleted document', now()), - ('20000000-0000-0000-0000-000000000001', '22222222-2222-2222-2222-222222222222', 'dddddddd-dddd-dddd-dddd-dddddddddddd', 'Org B draft', 'org B editor document', NULL); +INSERT INTO documents (id, org_id, owner_id, title, body) VALUES + ('10000000-0000-0000-0000-000000000001', '11111111-1111-1111-1111-111111111111', 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'Admin plan', 'org A admin document'), + ('10000000-0000-0000-0000-000000000002', '11111111-1111-1111-1111-111111111111', 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'Editor draft', 'org A editor document'), + ('20000000-0000-0000-0000-000000000001', '22222222-2222-2222-2222-222222222222', 'dddddddd-dddd-dddd-dddd-dddddddddddd', 'Org B draft', 'org B editor document'); From e59c7fa0b554c54a763e7b970d7ea233b1a5db12 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:08:32 +0000 Subject: [PATCH 04/10] chore: refresh eval results --- apps/web/src/data/eval-results.json | 365 +++++++++------------------- 1 file changed, 113 insertions(+), 252 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index cd410300..b0496344 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -688,7 +688,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -708,23 +708,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -734,14 +734,13 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" + "supabase" ] }, "docs": { "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-opus-5/build-rls-003-org-roles-permissions.json" @@ -2819,7 +2818,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -2839,23 +2838,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -2866,7 +2865,7 @@ "docs": { "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-opus-5-no-skills/build-rls-003-org-roles-permissions.json" @@ -4912,7 +4911,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -4932,23 +4931,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -4965,7 +4964,7 @@ "docs": { "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-rls-003-org-roles-permissions.json" @@ -6351,7 +6350,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -6371,23 +6370,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -6398,7 +6397,7 @@ "docs": { "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-003-org-roles-permissions.json" @@ -7923,7 +7922,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -7943,23 +7942,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -7977,35 +7976,35 @@ "calls": [ { "source": "search_docs", - "query": "query {\n searchDocs(query: \"row level security policies update select with check soft delete audit trigger\", limit: 5) {\n nodes {\n __typename\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "query": "query { searchDocs(query: \"row level security policies auth.uid select update with check members roles\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", + "title": "Build a User Management App with Refine" }, { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" }, { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" } ], - "resultChars": 77496 + "resultChars": 105199 } ] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/build-rls-003-org-roles-permissions.json" @@ -10390,7 +10389,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -10410,23 +10409,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -10435,122 +10434,9 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies security definer trigger soft delete audit log\", limit: 5) { nodes { title href content ... on Guide { subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#access-token", - "title": "Access token" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#authentication", - "title": "Authentication" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#authenticator-app", - "title": "Authenticator app" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#authorization", - "title": "Authorization" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#identity-provider", - "title": "Identity provider" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#json-web-token-jwt", - "title": "JSON Web Token (JWT)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#jwt-signing-secret", - "title": "JWT signing secret" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#multi-factor-authentication-mfa-or-2fa", - "title": "Multi-factor authentication (MFA or 2FA)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#nonce", - "title": "Nonce" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#oauth", - "title": "OAuth" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#oidc", - "title": "OIDC" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#one-time-password-otp", - "title": "One-time password (OTP)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#password-hashing-function", - "title": "Password hashing function" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#password-strength", - "title": "Password strength" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#pkce", - "title": "PKCE" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#provider-refresh-token", - "title": "Provider refresh token" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#provider-tokens", - "title": "Provider tokens" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token", - "title": "Refresh token" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#refresh-token-flow", - "title": "Refresh token flow" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#replay-attack", - "title": "Replay attack" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#row-level-security-policies-rls", - "title": "Row level security policies (RLS)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#saml", - "title": "SAML" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#session", - "title": "Session" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#single-sign-on-sso", - "title": "Single-sign on (SSO)" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary#time-based-one-time-password-totp", - "title": "Time-based one-time password (TOTP)" - } - ], - "resultChars": 20326 - } - ] + "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions.json" @@ -12889,7 +12775,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -12909,23 +12795,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -12941,6 +12827,34 @@ }, "docs": { "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security policies organization roles viewer editor admin auth.uid security definer\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/tables", + "title": "Tables and Data" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + } + ], + "resultChars": 96410 + }, { "source": "web_search", "query": "https://supabase.com/changelog.md", @@ -12952,24 +12866,12 @@ }, { "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking change RLS policies triggers audit soft delete", + "query": "site:supabase.com/changelog RLS breaking change Supabase changelog", "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security organization membership roles soft delete audit trigger security definer\", limit: 8) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - } - ], - "resultChars": 27158 } ] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6/build-rls-003-org-roles-permissions.json" @@ -15511,7 +15413,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -15531,23 +15433,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -15556,38 +15458,9 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres row level security auth.uid security definer policies triggers audit soft delete\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/architecture", - "title": "Auth architecture" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/triggers", - "title": "Postgres Triggers" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - } - ], - "resultChars": 56692 - } - ] + "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/build-rls-003-org-roles-permissions.json" @@ -17295,7 +17168,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -17315,23 +17188,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -17346,21 +17219,9 @@ ] }, "docs": { - "calls": [ - { - "source": "web_fetch", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 93795 - } - ] + "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "opencode-kimi-k3/build-rls-003-org-roles-permissions.json" @@ -18960,7 +18821,7 @@ "passed": true }, { - "name": "viewer sees active org documents only", + "name": "viewer sees only documents in their org", "passed": true }, { @@ -18980,23 +18841,23 @@ "passed": true }, { - "name": "admin can mark a document deleted in their org", + "name": "editor cannot delete another user's document", "passed": true }, { - "name": "documents are never hard-deleted", + "name": "admin can update any document in their org", "passed": true }, { - "name": "admin cannot affect another org", + "name": "admin can delete any document in their org", "passed": true }, { - "name": "WITH CHECK blocks editor from moving document to another org", + "name": "admin cannot affect another org", "passed": true }, { - "name": "write creates audit row with acting user", + "name": "WITH CHECK blocks editor from moving document to another org", "passed": true } ], @@ -19007,7 +18868,7 @@ "docs": { "calls": [] }, - "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org. Deletes should be recoverable, and we want to know who changed what.", + "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-rls-003-org-roles-permissions.json" From f8cc627fe60df636b9f2f110a5ea17000598db38 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:33:13 -0400 Subject: [PATCH 05/10] refactor: rewrite build-rls-003 scorer with named check functions Matches the checkX(ctx) pattern used elsewhere in the repo instead of imperative checks.push() calls. Also normalizes ctx.query into a non-throwing runAsUser helper so an RLS-blocked write (a real Postgres error) is a value each check judges, not an exception that silently aborts every check after it. --- .../EVAL.ts | 481 ++++++++++-------- 1 file changed, 274 insertions(+), 207 deletions(-) diff --git a/evals/build-rls-003-org-roles-permissions/EVAL.ts b/evals/build-rls-003-org-roles-permissions/EVAL.ts index ff865d69..3150fe9d 100644 --- a/evals/build-rls-003-org-roles-permissions/EVAL.ts +++ b/evals/build-rls-003-org-roles-permissions/EVAL.ts @@ -1,4 +1,8 @@ -import type { CheckResult, ToolScorer } from '@supabase-evals/core'; +import type { + CheckResult, + ToolEvalContext, + ToolScorer, +} from '@supabase-evals/core'; import { stripIndent } from 'common-tags'; const ORG_A = '11111111-1111-1111-1111-111111111111'; @@ -7,223 +11,286 @@ const ADMIN_A = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'; const EDITOR_A = 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'; const VIEWER_A = 'cccccccc-cccc-cccc-cccc-cccccccccccc'; -const asUser = ( +const scorer: ToolScorer = async (ctx) => { + try { + const checks: CheckResult[] = [ + await checkRlsEnabled(ctx), + await checkViewerSeesOnlyOwnOrgDocuments(ctx), + await checkViewerCannotInsert(ctx), + await checkEditorCanInsertOwnOrgDocument(ctx), + await checkEditorCanUpdateOwnDocument(ctx), + await checkEditorCannotUpdateAnotherUsersDocument(ctx), + await checkEditorCannotDeleteAnotherUsersDocument(ctx), + await checkAdminCanUpdateAnyDocumentInOrg(ctx), + await checkAdminCanDeleteAnyDocumentInOrg(ctx), + await checkAdminCannotAffectAnotherOrg(ctx), + await checkWithCheckBlocksOrgReassignment(ctx), + await checkCannotSeeAnotherOrgsMembershipRoster(ctx), + ]; + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { name: 'scorer evaluated org role RLS', passed: false, notes: msg }, + ], + }; + } +}; + +export default scorer; + +/** Builds the SQL to run a query as a given user via forged JWT claims, the same session state PostgREST sets for a real request. */ +function asUser( sub: string, body: string, finish: 'COMMIT' | 'ROLLBACK' = 'COMMIT' -) => stripIndent` - BEGIN; - SET LOCAL ROLE authenticated; - SET LOCAL request.jwt.claim.sub = '${sub}'; - SET LOCAL request.jwt.claim.role = 'authenticated'; - ${body} - ${finish}; -`; +): string { + return stripIndent` + BEGIN; + SET LOCAL ROLE authenticated; + SET LOCAL request.jwt.claim.sub = '${sub}'; + SET LOCAL request.jwt.claim.role = 'authenticated'; + ${body} + ${finish}; + `; +} -const scorer: ToolScorer = async (ctx) => { - const q = (sql: string) => ctx.query(sql); - const checks: CheckResult[] = []; - - const resetTx = async () => { - try { - await q('ROLLBACK;'); - } catch { - // Clear aborted scorer transactions. - } - }; +type UserQueryResult = { rows: Record[]; error: Error | null }; +/** Runs a query as a user without throwing, so RLS-blocked writes (WITH CHECK violations, revoked grants) are a value each check can judge, not an exception that aborts every check after it. */ +async function runAsUser( + ctx: ToolEvalContext, + sub: string, + body: string, + finish: 'COMMIT' | 'ROLLBACK' = 'COMMIT' +): Promise { try { - const { rows: rls } = await q( - `SELECT relrowsecurity FROM pg_class WHERE relname = 'documents';` - ); - checks.push({ - name: 'RLS enabled on documents', - passed: rls[0]?.relrowsecurity === true, - }); - - const { rows: viewerReads } = await q( - asUser(VIEWER_A, `SELECT title FROM documents ORDER BY title;`) - ); - checks.push({ - name: 'viewer sees only documents in their org', - passed: - viewerReads.length === 2 && - viewerReads.map((row) => row.title).join(',') === - 'Admin plan,Editor draft', - }); - - let viewerInsertBlocked = false; - try { - await q( - asUser( - VIEWER_A, - stripIndent` - INSERT INTO documents (org_id, owner_id, title, body) - VALUES ('${ORG_A}', '${VIEWER_A}', 'viewer insert', 'should fail'); - ` - ) - ); - } catch { - viewerInsertBlocked = true; - await resetTx(); - } - checks.push({ name: 'viewer cannot insert', passed: viewerInsertBlocked }); - - const { rows: editorInsert } = await q( - asUser( - EDITOR_A, - stripIndent` - INSERT INTO documents (org_id, owner_id, title, body) - VALUES ('${ORG_A}', '${EDITOR_A}', 'editor insert', 'allowed') - RETURNING id; - `, - 'ROLLBACK' - ) - ); - checks.push({ - name: 'editor can insert own org document', - passed: editorInsert.length === 1, - }); - - const { rows: editorOwnUpdate } = await q( - asUser( - EDITOR_A, - stripIndent` - UPDATE documents - SET body = 'editor changed own document' - WHERE id = '10000000-0000-0000-0000-000000000002' - RETURNING id; - `, - 'ROLLBACK' - ) - ); - checks.push({ - name: 'editor can update own document', - passed: editorOwnUpdate.length === 1, - }); - - const { rows: editorUpdatesAdmin } = await q( - asUser( - EDITOR_A, - stripIndent` - UPDATE documents - SET body = 'editor changed admin document' - WHERE id = '10000000-0000-0000-0000-000000000001' - RETURNING id; - `, - 'ROLLBACK' - ) - ); - checks.push({ - name: "editor cannot update another user's document", - passed: editorUpdatesAdmin.length === 0, - }); - - let editorDeleteBlocked = false; - try { - const { rows } = await q( - asUser( - EDITOR_A, - `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000001' RETURNING id;`, - 'ROLLBACK' - ) - ); - editorDeleteBlocked = rows.length === 0; - } catch { - editorDeleteBlocked = true; - await resetTx(); - } - checks.push({ - name: "editor cannot delete another user's document", - passed: editorDeleteBlocked, - }); - - const { rows: adminUpdate } = await q( - asUser( - ADMIN_A, - stripIndent` - UPDATE documents - SET body = 'admin changed editor document' - WHERE id = '10000000-0000-0000-0000-000000000002' - RETURNING id; - `, - 'ROLLBACK' - ) - ); - checks.push({ - name: 'admin can update any document in their org', - passed: adminUpdate.length === 1, - }); - - const { rows: adminDelete } = await q( - asUser( - ADMIN_A, - `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000002' RETURNING id;`, - 'ROLLBACK' - ) - ); - checks.push({ - name: 'admin can delete any document in their org', - passed: adminDelete.length === 1, - }); - - const { rows: adminCrossOrg } = await q( - asUser( - ADMIN_A, - stripIndent` - UPDATE documents - SET body = 'admin touched another org' - WHERE id = '20000000-0000-0000-0000-000000000001' - RETURNING id; - `, - 'ROLLBACK' - ) - ); - checks.push({ - name: 'admin cannot affect another org', - passed: adminCrossOrg.length === 0, - }); - - let orgReassignmentBlocked = false; - try { - const { rows } = await q( - asUser( - EDITOR_A, - stripIndent` - UPDATE documents - SET org_id = '${ORG_B}' - WHERE id = '10000000-0000-0000-0000-000000000002' - RETURNING id; - `, - 'ROLLBACK' - ) - ); - orgReassignmentBlocked = rows.length === 0; - } catch { - orgReassignmentBlocked = true; - await resetTx(); - } - checks.push({ - name: 'WITH CHECK blocks editor from moving document to another org', - passed: orgReassignmentBlocked, - }); + const { rows } = await ctx.query(asUser(sub, body, finish)); + return { rows, error: null }; } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - checks.push({ - name: 'scorer evaluated org role RLS', - passed: false, - notes: msg, - }); + await ctx.query('ROLLBACK;').catch(() => {}); return { - passed: false, - checks, + rows: [], + error: error instanceof Error ? error : new Error(String(error)), }; } +} +/** Checks that RLS is turned on for the documents table at all. */ +async function checkRlsEnabled(ctx: ToolEvalContext): Promise { + const { rows } = await ctx.query( + `SELECT relrowsecurity FROM pg_class WHERE relname = 'documents';` + ); return { - passed: checks.every((check) => check.passed), - checks, + name: 'RLS enabled on documents', + passed: rows[0]?.relrowsecurity === true, }; -}; +} -export default scorer; +/** Checks that a viewer only sees active documents in their own org. */ +async function checkViewerSeesOnlyOwnOrgDocuments( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + VIEWER_A, + `SELECT title FROM documents ORDER BY title;` + ); + return { + name: 'viewer sees only documents in their org', + passed: + !result.error && + result.rows.length === 2 && + result.rows.map((row) => row.title).join(',') === + 'Admin plan,Editor draft', + }; +} + +/** Checks that a viewer cannot insert a new document. */ +async function checkViewerCannotInsert( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + VIEWER_A, + stripIndent` + INSERT INTO documents (org_id, owner_id, title, body) + VALUES ('${ORG_A}', '${VIEWER_A}', 'viewer insert', 'should fail'); + ` + ); + return { name: 'viewer cannot insert', passed: Boolean(result.error) }; +} + +/** Checks that an editor can insert a document they own in their org. */ +async function checkEditorCanInsertOwnOrgDocument( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + EDITOR_A, + stripIndent` + INSERT INTO documents (org_id, owner_id, title, body) + VALUES ('${ORG_A}', '${EDITOR_A}', 'editor insert', 'allowed') + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: 'editor can insert own org document', + passed: !result.error && result.rows.length === 1, + }; +} + +/** Checks that an editor can update a document they own. */ +async function checkEditorCanUpdateOwnDocument( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + EDITOR_A, + stripIndent` + UPDATE documents + SET body = 'editor changed own document' + WHERE id = '10000000-0000-0000-0000-000000000002' + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: 'editor can update own document', + passed: !result.error && result.rows.length === 1, + }; +} + +/** Checks that an editor cannot update a document owned by someone else. */ +async function checkEditorCannotUpdateAnotherUsersDocument( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + EDITOR_A, + stripIndent` + UPDATE documents + SET body = 'editor changed admin document' + WHERE id = '10000000-0000-0000-0000-000000000001' + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: "editor cannot update another user's document", + passed: Boolean(result.error) || result.rows.length === 0, + }; +} + +/** Checks that an editor cannot delete a document owned by someone else. */ +async function checkEditorCannotDeleteAnotherUsersDocument( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + EDITOR_A, + `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000001' RETURNING id;`, + 'ROLLBACK' + ); + return { + name: "editor cannot delete another user's document", + passed: Boolean(result.error) || result.rows.length === 0, + }; +} + +/** Checks that an admin can update any document in their org, not just their own. */ +async function checkAdminCanUpdateAnyDocumentInOrg( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + ADMIN_A, + stripIndent` + UPDATE documents + SET body = 'admin changed editor document' + WHERE id = '10000000-0000-0000-0000-000000000002' + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: 'admin can update any document in their org', + passed: !result.error && result.rows.length === 1, + }; +} + +/** Checks that an admin can delete any document in their org, not just their own. */ +async function checkAdminCanDeleteAnyDocumentInOrg( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + ADMIN_A, + `DELETE FROM documents WHERE id = '10000000-0000-0000-0000-000000000002' RETURNING id;`, + 'ROLLBACK' + ); + return { + name: 'admin can delete any document in their org', + passed: !result.error && result.rows.length === 1, + }; +} + +/** Checks that an admin cannot affect documents in a different org. */ +async function checkAdminCannotAffectAnotherOrg( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + ADMIN_A, + stripIndent` + UPDATE documents + SET body = 'admin touched another org' + WHERE id = '20000000-0000-0000-0000-000000000001' + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: 'admin cannot affect another org', + passed: Boolean(result.error) || result.rows.length === 0, + }; +} + +/** Checks that WITH CHECK blocks an editor from moving a document to another org. */ +async function checkWithCheckBlocksOrgReassignment( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + EDITOR_A, + stripIndent` + UPDATE documents + SET org_id = '${ORG_B}' + WHERE id = '10000000-0000-0000-0000-000000000002' + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: 'WITH CHECK blocks editor from moving document to another org', + passed: Boolean(result.error) || result.rows.length === 0, + }; +} + +/** Checks that a member of one org cannot see another org's membership roster. */ +async function checkCannotSeeAnotherOrgsMembershipRoster( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + EDITOR_A, + `SELECT user_id FROM memberships WHERE org_id = '${ORG_B}';` + ); + return { + name: "cannot see another org's membership roster", + passed: Boolean(result.error) || result.rows.length === 0, + }; +} From a692c5c62a768a3af56911436a4771c3ae8a77bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:44:21 +0000 Subject: [PATCH 06/10] chore: refresh eval results --- apps/web/src/data/eval-results.json | 161 ++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 34 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index b0496344..1904d539 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -726,6 +726,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -2856,6 +2860,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -4949,6 +4957,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -6388,6 +6400,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -7960,6 +7976,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -7976,13 +7996,9 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies auth.uid select update with check members roles\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"row level security policies update requires select policy TO authenticated security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content href } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" @@ -7992,15 +8008,47 @@ "title": "Build a User Management App with Refine" }, { - "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", - "title": "Column Level Security" + "url": "https://supabase.com/docs/guides/database/tables", + "title": "Tables and Data" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-hooks", + "title": "Auth Hooks" + } + ], + "resultChars": 127190 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"row level security organization role admin editor viewer docs\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", + "title": "SSO Testing and Best Practices" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + }, + { + "url": "https://supabase.com/docs/guides/database/tables", + "title": "Tables and Data" } ], - "resultChars": 105199 + "resultChars": 99167 } ] }, @@ -10427,6 +10475,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -10438,7 +10490,7 @@ }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions.json" }, { @@ -12813,6 +12865,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -12827,47 +12883,43 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- security references ---'; rg --files .agents/skills/supabase-postgres-best-practices/references | rg '/security-|query-missing-indexes|schema-foreign' ; printf '%s\\\\n' '--- changelog security tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|row.level|rls|policy'\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5114 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies organization roles viewer editor admin auth.uid security definer\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Row Level Security policies SELECT INSERT UPDATE DELETE auth.uid organization membership role viewer editor admin\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and Data" - }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, { "url": "https://supabase.com/docs/guides/resources/glossary", "title": "Glossary" }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" } ], - "resultChars": 96410 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog RLS breaking change Supabase changelog", - "pages": [] + "resultChars": 77235 } ] }, @@ -15451,6 +15503,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -15458,7 +15514,36 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres Row Level Security security definer helper function policies auth.uid memberships recursion\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" + }, + { + "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", + "title": "Database Advisor: Lint 0003_auth_rls_initplan" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + } + ], + "resultChars": 77036 + } + ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", @@ -17206,6 +17291,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { @@ -18859,6 +18948,10 @@ { "name": "WITH CHECK blocks editor from moving document to another org", "passed": true + }, + { + "name": "cannot see another org's membership roster", + "passed": true } ], "skills": { From 3f551c5e93d71f15dc0d497bd37b1015f736bd46 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:13:08 -0400 Subject: [PATCH 07/10] test: add multi-org role-scoping checks to build-rls-003 Seeds a user who's a viewer in org A and admin in org B, then checks that their org-B admin role doesn't leak into org A. Every seeded user previously belonged to exactly one org, so a policy that checked role without joining on org_id would have passed undetected. All tested benchmark agents still pass, but it's real coverage for a common multi-tenant RLS mistake. --- .../EVAL.ts | 45 +++++++++++++++++++ .../remote/project.sql | 4 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/evals/build-rls-003-org-roles-permissions/EVAL.ts b/evals/build-rls-003-org-roles-permissions/EVAL.ts index 3150fe9d..82a63534 100644 --- a/evals/build-rls-003-org-roles-permissions/EVAL.ts +++ b/evals/build-rls-003-org-roles-permissions/EVAL.ts @@ -10,6 +10,7 @@ const ORG_B = '22222222-2222-2222-2222-222222222222'; const ADMIN_A = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'; const EDITOR_A = 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'; const VIEWER_A = 'cccccccc-cccc-cccc-cccc-cccccccccccc'; +const MULTI_ORG_USER = 'eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee'; const scorer: ToolScorer = async (ctx) => { try { @@ -26,6 +27,8 @@ const scorer: ToolScorer = async (ctx) => { await checkAdminCannotAffectAnotherOrg(ctx), await checkWithCheckBlocksOrgReassignment(ctx), await checkCannotSeeAnotherOrgsMembershipRoster(ctx), + await checkOtherOrgRoleDoesNotLeakIn(ctx), + await checkOwnOrgAdminRoleStillWorks(ctx), ]; return { passed: checks.every((c) => c.passed), checks }; } catch (error) { @@ -294,3 +297,45 @@ async function checkCannotSeeAnotherOrgsMembershipRoster( passed: Boolean(result.error) || result.rows.length === 0, }; } + +/** Checks that a user who is admin in org B can't use that role to write in org A, where they're only a viewer. Role lives on the (user_id, org_id) row, not the user, so this only holds if the org_id join was actually applied. */ +async function checkOtherOrgRoleDoesNotLeakIn( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + MULTI_ORG_USER, + stripIndent` + UPDATE documents + SET body = 'multi-org user tried to act as admin here' + WHERE id = '10000000-0000-0000-0000-000000000001' + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: "a viewer role in one org doesn't grant admin power in another org", + passed: Boolean(result.error) || result.rows.length === 0, + }; +} + +/** Checks that the same multi-org user's admin role still works normally in the org where they actually hold it. */ +async function checkOwnOrgAdminRoleStillWorks( + ctx: ToolEvalContext +): Promise { + const result = await runAsUser( + ctx, + MULTI_ORG_USER, + stripIndent` + UPDATE documents + SET body = 'multi-org user acting as admin in their own org' + WHERE id = '20000000-0000-0000-0000-000000000001' + RETURNING id; + `, + 'ROLLBACK' + ); + return { + name: 'multi-org user can act as admin in the org where they hold that role', + passed: !result.error && result.rows.length === 1, + }; +} diff --git a/evals/build-rls-003-org-roles-permissions/remote/project.sql b/evals/build-rls-003-org-roles-permissions/remote/project.sql index efdcb524..79ed8bed 100644 --- a/evals/build-rls-003-org-roles-permissions/remote/project.sql +++ b/evals/build-rls-003-org-roles-permissions/remote/project.sql @@ -26,7 +26,9 @@ INSERT INTO memberships (user_id, org_id, role) VALUES ('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', '11111111-1111-1111-1111-111111111111', 'admin'), ('bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', '11111111-1111-1111-1111-111111111111', 'editor'), ('cccccccc-cccc-cccc-cccc-cccccccccccc', '11111111-1111-1111-1111-111111111111', 'viewer'), - ('dddddddd-dddd-dddd-dddd-dddddddddddd', '22222222-2222-2222-2222-222222222222', 'editor'); + ('dddddddd-dddd-dddd-dddd-dddddddddddd', '22222222-2222-2222-2222-222222222222', 'editor'), + ('eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', '11111111-1111-1111-1111-111111111111', 'viewer'), + ('eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', '22222222-2222-2222-2222-222222222222', 'admin'); INSERT INTO documents (id, org_id, owner_id, title, body) VALUES ('10000000-0000-0000-0000-000000000001', '11111111-1111-1111-1111-111111111111', 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'Admin plan', 'org A admin document'), From 4054183b5c43938e5d0f3e00c782faa6086bd9ef Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:16:42 -0400 Subject: [PATCH 08/10] docs: trim and cite comments in build-rls-003 scorer Fixed a stale "active documents" reference left over from the removed soft-delete concept, and swapped an unverified PostgREST claim for a cited Supabase testing-guide reference. --- evals/build-rls-003-org-roles-permissions/EVAL.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/evals/build-rls-003-org-roles-permissions/EVAL.ts b/evals/build-rls-003-org-roles-permissions/EVAL.ts index 82a63534..4a7ec02f 100644 --- a/evals/build-rls-003-org-roles-permissions/EVAL.ts +++ b/evals/build-rls-003-org-roles-permissions/EVAL.ts @@ -44,7 +44,7 @@ const scorer: ToolScorer = async (ctx) => { export default scorer; -/** Builds the SQL to run a query as a given user via forged JWT claims, the same session state PostgREST sets for a real request. */ +/** Builds the SQL to run a query as a given user. Same technique as Supabase's own RLS testing guide (https://supabase.com/docs/guides/local-development/testing/overview). */ function asUser( sub: string, body: string, @@ -62,7 +62,7 @@ function asUser( type UserQueryResult = { rows: Record[]; error: Error | null }; -/** Runs a query as a user without throwing, so RLS-blocked writes (WITH CHECK violations, revoked grants) are a value each check can judge, not an exception that aborts every check after it. */ +/** Runs a query as a user without throwing. RLS-blocked writes (WITH CHECK violations, revoked grants) become a value each check judges, so one blocked write can't abort every later check. */ async function runAsUser( ctx: ToolEvalContext, sub: string, @@ -81,7 +81,7 @@ async function runAsUser( } } -/** Checks that RLS is turned on for the documents table at all. */ +/** Checks that RLS is turned on for the documents table. */ async function checkRlsEnabled(ctx: ToolEvalContext): Promise { const { rows } = await ctx.query( `SELECT relrowsecurity FROM pg_class WHERE relname = 'documents';` @@ -92,7 +92,7 @@ async function checkRlsEnabled(ctx: ToolEvalContext): Promise { }; } -/** Checks that a viewer only sees active documents in their own org. */ +/** Checks that a viewer only sees documents in their own org. */ async function checkViewerSeesOnlyOwnOrgDocuments( ctx: ToolEvalContext ): Promise { @@ -298,7 +298,7 @@ async function checkCannotSeeAnotherOrgsMembershipRoster( }; } -/** Checks that a user who is admin in org B can't use that role to write in org A, where they're only a viewer. Role lives on the (user_id, org_id) row, not the user, so this only holds if the org_id join was actually applied. */ +/** Checks that a user's admin role in org B doesn't leak into org A, where they're only a viewer. Role lives on the (user_id, org_id) row, not the user, so this only holds if policies actually join on org_id. */ async function checkOtherOrgRoleDoesNotLeakIn( ctx: ToolEvalContext ): Promise { @@ -319,7 +319,7 @@ async function checkOtherOrgRoleDoesNotLeakIn( }; } -/** Checks that the same multi-org user's admin role still works normally in the org where they actually hold it. */ +/** Checks that the same multi-org user's admin role still works in the org where they hold it. */ async function checkOwnOrgAdminRoleStillWorks( ctx: ToolEvalContext ): Promise { From fb8e87089f0e721faf6d97fd4b9c8a68351e3154 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:36:33 +0000 Subject: [PATCH 09/10] chore: refresh eval results --- apps/web/src/data/eval-results.json | 206 +++++++++++++++++----------- 1 file changed, 129 insertions(+), 77 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 1904d539..8fa43a97 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -730,6 +730,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -2864,6 +2872,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -4961,6 +4977,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -6404,6 +6428,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -7980,6 +8012,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -7996,59 +8036,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"row level security policies update requires select policy TO authenticated security invoker\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } ... on ClientLibraryFunctionReference { title href methodName language content href } } } }", + "query": "query { searchDocs(query: \"row level security organization admin editor viewer shared documents policies\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-refine", - "title": "Build a User Management App with Refine" - }, - { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and Data" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" + "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", + "title": "Advanced pgTAP Testing" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks", - "title": "Auth Hooks" - } - ], - "resultChars": 127190 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"row level security organization role admin editor viewer docs\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" }, { "url": "https://supabase.com/docs/guides/getting-started/features", "title": "Features" }, { - "url": "https://supabase.com/docs/guides/platform/sso/testing-best-practices", - "title": "SSO Testing and Best Practices" - }, - { - "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", - "title": "Multiple SSO Providers" + "url": "https://supabase.com/docs/guides/storage/buckets/fundamentals", + "title": "Storage Buckets" }, { - "url": "https://supabase.com/docs/guides/database/tables", - "title": "Tables and Data" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" } ], - "resultChars": 99167 + "resultChars": 59205 } ] }, @@ -10479,6 +10491,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -10486,11 +10506,40 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"row level security policies memberships auth.uid organization role\", limit: 5) {\n nodes {\n ... on Guide {\n title\n href\n content\n }\n ... on TroubleshootingGuide {\n title\n href\n content\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", + "title": "Column Level Security" + }, + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" + }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + } + ], + "resultChars": 75044 + } + ] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-003-org-roles-permissions.json" }, { @@ -12869,6 +12918,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -12885,17 +12942,17 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- security references ---'; rg --files .agents/skills/supabase-postgres-best-practices/references | rg '/security-|query-missing-indexes|schema-foreign' ; printf '%s\\\\n' '--- changelog security tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -n -m 20 'breaking-change|row.level|rls|policy'\"", + "query": "/bin/bash -lc \"find .claude/skills/supabase-postgres-best-practices/references -maxdepth 1 -type f | sort | sed -n '/security/p' && curl -fsSL https://supabase.com/changelog.md | rg -n -i 'breaking-change|rls|row level|policy' | head -40\"", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5114 + "resultChars": 8612 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security policies SELECT INSERT UPDATE DELETE auth.uid organization membership role viewer editor admin\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Row Level Security policies auth.uid organization membership SELECT UPDATE INSERT DELETE security definer\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { @@ -12906,14 +12963,14 @@ "url": "https://supabase.com/docs/guides/database/postgres/column-level-security", "title": "Column Level Security" }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - }, { "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", "title": "Row Level Security" }, + { + "url": "https://supabase.com/docs/guides/resources/glossary", + "title": "Glossary" + }, { "url": "https://supabase.com/docs/guides/api/securing-your-api", "title": "Securing your API" @@ -15507,6 +15564,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -15514,36 +15579,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres Row Level Security security definer helper function policies auth.uid memberships recursion\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0003_auth_rls_initplan", - "title": "Database Advisor: Lint 0003_auth_rls_initplan" - }, - { - "url": "https://supabase.com/docs/guides/resources/glossary", - "title": "Glossary" - } - ], - "resultChars": 77036 - } - ] + "calls": [] }, "prompt": "Access control on our shared docs feature needs work, people can see and edit documents they shouldn't. Viewers should just be able to read, editors should only manage their own docs, and admins should be able to manage anything in their org.", "promptSourcePath": "evals/build-rls-003-org-roles-permissions/PROMPT.md", @@ -17295,6 +17331,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { @@ -18952,6 +18996,14 @@ { "name": "cannot see another org's membership roster", "passed": true + }, + { + "name": "a viewer role in one org doesn't grant admin power in another org", + "passed": true + }, + { + "name": "multi-org user can act as admin in the org where they hold that role", + "passed": true } ], "skills": { From 7a451692312ce01ade0d20e28a17b7142d021612 Mon Sep 17 00:00:00 2001 From: Matt Rossman <22670878+mattrossman@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:38:43 -0400 Subject: [PATCH 10/10] docs: clean up scorer comments in build-rls-003 Removes check comments that just restated their function name, merges asUser into runAsUser since the split had no independent caller, and clarifies the multi-org role-leak comment to name the memberships table explicitly. --- .../EVAL.ts | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/evals/build-rls-003-org-roles-permissions/EVAL.ts b/evals/build-rls-003-org-roles-permissions/EVAL.ts index 4a7ec02f..dfc32011 100644 --- a/evals/build-rls-003-org-roles-permissions/EVAL.ts +++ b/evals/build-rls-003-org-roles-permissions/EVAL.ts @@ -44,25 +44,15 @@ const scorer: ToolScorer = async (ctx) => { export default scorer; -/** Builds the SQL to run a query as a given user. Same technique as Supabase's own RLS testing guide (https://supabase.com/docs/guides/local-development/testing/overview). */ -function asUser( - sub: string, - body: string, - finish: 'COMMIT' | 'ROLLBACK' = 'COMMIT' -): string { - return stripIndent` - BEGIN; - SET LOCAL ROLE authenticated; - SET LOCAL request.jwt.claim.sub = '${sub}'; - SET LOCAL request.jwt.claim.role = 'authenticated'; - ${body} - ${finish}; - `; -} - type UserQueryResult = { rows: Record[]; error: Error | null }; -/** Runs a query as a user without throwing. RLS-blocked writes (WITH CHECK violations, revoked grants) become a value each check judges, so one blocked write can't abort every later check. */ +/** + * Runs a query as a given user, via the request.jwt.claim.sub/role + * technique shown in RLS testing guide: + * (https://supabase.com/docs/guides/local-development/testing/overview). + * + * Returns errors (incl. RLS-blocked writes) as values instead of throwing. + */ async function runAsUser( ctx: ToolEvalContext, sub: string, @@ -70,7 +60,14 @@ async function runAsUser( finish: 'COMMIT' | 'ROLLBACK' = 'COMMIT' ): Promise { try { - const { rows } = await ctx.query(asUser(sub, body, finish)); + const { rows } = await ctx.query(stripIndent` + BEGIN; + SET LOCAL ROLE authenticated; + SET LOCAL request.jwt.claim.sub = '${sub}'; + SET LOCAL request.jwt.claim.role = 'authenticated'; + ${body} + ${finish}; + `); return { rows, error: null }; } catch (error) { await ctx.query('ROLLBACK;').catch(() => {}); @@ -81,7 +78,6 @@ async function runAsUser( } } -/** Checks that RLS is turned on for the documents table. */ async function checkRlsEnabled(ctx: ToolEvalContext): Promise { const { rows } = await ctx.query( `SELECT relrowsecurity FROM pg_class WHERE relname = 'documents';` @@ -92,7 +88,6 @@ async function checkRlsEnabled(ctx: ToolEvalContext): Promise { }; } -/** Checks that a viewer only sees documents in their own org. */ async function checkViewerSeesOnlyOwnOrgDocuments( ctx: ToolEvalContext ): Promise { @@ -111,7 +106,6 @@ async function checkViewerSeesOnlyOwnOrgDocuments( }; } -/** Checks that a viewer cannot insert a new document. */ async function checkViewerCannotInsert( ctx: ToolEvalContext ): Promise { @@ -126,7 +120,6 @@ async function checkViewerCannotInsert( return { name: 'viewer cannot insert', passed: Boolean(result.error) }; } -/** Checks that an editor can insert a document they own in their org. */ async function checkEditorCanInsertOwnOrgDocument( ctx: ToolEvalContext ): Promise { @@ -146,7 +139,6 @@ async function checkEditorCanInsertOwnOrgDocument( }; } -/** Checks that an editor can update a document they own. */ async function checkEditorCanUpdateOwnDocument( ctx: ToolEvalContext ): Promise { @@ -167,7 +159,6 @@ async function checkEditorCanUpdateOwnDocument( }; } -/** Checks that an editor cannot update a document owned by someone else. */ async function checkEditorCannotUpdateAnotherUsersDocument( ctx: ToolEvalContext ): Promise { @@ -188,7 +179,6 @@ async function checkEditorCannotUpdateAnotherUsersDocument( }; } -/** Checks that an editor cannot delete a document owned by someone else. */ async function checkEditorCannotDeleteAnotherUsersDocument( ctx: ToolEvalContext ): Promise { @@ -204,7 +194,6 @@ async function checkEditorCannotDeleteAnotherUsersDocument( }; } -/** Checks that an admin can update any document in their org, not just their own. */ async function checkAdminCanUpdateAnyDocumentInOrg( ctx: ToolEvalContext ): Promise { @@ -225,7 +214,6 @@ async function checkAdminCanUpdateAnyDocumentInOrg( }; } -/** Checks that an admin can delete any document in their org, not just their own. */ async function checkAdminCanDeleteAnyDocumentInOrg( ctx: ToolEvalContext ): Promise { @@ -241,7 +229,6 @@ async function checkAdminCanDeleteAnyDocumentInOrg( }; } -/** Checks that an admin cannot affect documents in a different org. */ async function checkAdminCannotAffectAnotherOrg( ctx: ToolEvalContext ): Promise { @@ -262,7 +249,6 @@ async function checkAdminCannotAffectAnotherOrg( }; } -/** Checks that WITH CHECK blocks an editor from moving a document to another org. */ async function checkWithCheckBlocksOrgReassignment( ctx: ToolEvalContext ): Promise { @@ -283,7 +269,6 @@ async function checkWithCheckBlocksOrgReassignment( }; } -/** Checks that a member of one org cannot see another org's membership roster. */ async function checkCannotSeeAnotherOrgsMembershipRoster( ctx: ToolEvalContext ): Promise { @@ -298,7 +283,13 @@ async function checkCannotSeeAnotherOrgsMembershipRoster( }; } -/** Checks that a user's admin role in org B doesn't leak into org A, where they're only a viewer. Role lives on the (user_id, org_id) row, not the user, so this only holds if policies actually join on org_id. */ +/** + * Checks that a user's admin role in org B doesn't leak into org A, where + * they're only a viewer. In the memberships table, role is a column on the + * (user_id, org_id) row, not the user, so the same user can hold a + * different role per org. This checks if policies actually join on + * org_id instead of just checking the user's role anywhere. + */ async function checkOtherOrgRoleDoesNotLeakIn( ctx: ToolEvalContext ): Promise { @@ -319,7 +310,6 @@ async function checkOtherOrgRoleDoesNotLeakIn( }; } -/** Checks that the same multi-org user's admin role still works in the org where they hold it. */ async function checkOwnOrgAdminRoleStillWorks( ctx: ToolEvalContext ): Promise {