From 86bddd5fcb0c97f93e8e4931495d3715a08e1ac3 Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Mon, 27 Jul 2026 18:20:18 +0300 Subject: [PATCH 1/8] feat: supabase-js SDK evals (auth flow, relational report, key migration) --- .gitignore | 2 + .../EVAL.ts | 238 ++++++++++++++++++ .../PROMPT.md | 31 +++ .../README.md | 19 ++ .../local/app/package.json | 5 + .../local/app/src/auth.mjs | 36 +++ .../local/supabase/config.toml | 165 ++++++++++++ .../migrations/0000_profiles_schema.sql | 53 ++++ .../EVAL.ts | 214 ++++++++++++++++ .../PROMPT.md | 27 ++ .../local/app/package.json | 5 + .../local/app/report.mjs | 22 ++ .../local/supabase/config.toml | 165 ++++++++++++ .../migrations/0000_orders_schema.sql | 69 +++++ .../EVAL.ts | 230 +++++++++++++++++ .../PROMPT.md | 26 ++ .../README.md | 24 ++ .../local/app/.env | 4 + .../local/app/package.json | 12 + .../local/app/posts.mjs | 19 ++ .../local/app/stats.mjs | 19 ++ .../local/supabase/config.toml | 165 ++++++++++++ .../supabase/migrations/0000_posts_schema.sql | 27 ++ 23 files changed, 1577 insertions(+) create mode 100644 evals/build-auth-001-email-password-flow/EVAL.ts create mode 100644 evals/build-auth-001-email-password-flow/PROMPT.md create mode 100644 evals/build-auth-001-email-password-flow/README.md create mode 100644 evals/build-auth-001-email-password-flow/local/app/package.json create mode 100644 evals/build-auth-001-email-password-flow/local/app/src/auth.mjs create mode 100644 evals/build-auth-001-email-password-flow/local/supabase/config.toml create mode 100644 evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql create mode 100644 evals/build-dataapi-001-relational-report/EVAL.ts create mode 100644 evals/build-dataapi-001-relational-report/PROMPT.md create mode 100644 evals/build-dataapi-001-relational-report/local/app/package.json create mode 100644 evals/build-dataapi-001-relational-report/local/app/report.mjs create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/config.toml create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql create mode 100644 evals/resolve-sdk-001-legacy-key-migration/EVAL.ts create mode 100644 evals/resolve-sdk-001-legacy-key-migration/PROMPT.md create mode 100644 evals/resolve-sdk-001-legacy-key-migration/README.md create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/.env create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/package.json create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql diff --git a/.gitignore b/.gitignore index 932a6aea..23951d46 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ node_modules/ dist/ .env .env*.local +# eval seed data may include a .env with well-known local demo keys +!evals/*/local/**/.env .DS_Store results/*/ .sync-tmp/ diff --git a/evals/build-auth-001-email-password-flow/EVAL.ts b/evals/build-auth-001-email-password-flow/EVAL.ts new file mode 100644 index 00000000..f658d512 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/EVAL.ts @@ -0,0 +1,238 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Email+password auth benchmark: the prompt asks the agent to finish the +// app's auth layer (app/src/auth.mjs) against the running local stack and +// never names supabase-js — the "uses @supabase/supabase-js" check is +// GATING. On top of SDK discovery it measures driving auth correctly: +// passing the display name as signup user metadata (the seeded profiles +// trigger reads raw_user_meta_data), handling bad credentials gracefully, +// and reading the RLS-scoped profile with the session actually attached. + +const APP_DIR = 'app'; +const DRIVER = 'eval-driver.mjs'; +const DRIVER_MARKER = '___EVAL_DRIVER___'; + +// Runs inside the sandbox, in one process, exactly like the app would use +// the module: sign up, fail a sign-in, sign in, read the profile. +const DRIVER_SOURCE = ` +import { signUp, signIn, getMyProfile } from './src/auth.mjs'; + +const [email, password, displayName, wrongPassword] = process.argv.slice(2); +const out = {}; +const step = async (name, fn) => { + try { + out[name] = await fn(); + } catch (error) { + out[name] = { + threw: String(error instanceof Error ? error.message : error), + }; + } +}; +await step('signUp', () => signUp(email, password, displayName)); +await step('signInWrong', () => signIn(email, wrongPassword)); +await step('signIn', () => signIn(email, password)); +await step('profile', () => getMyProfile()); +console.log('${DRIVER_MARKER}' + JSON.stringify(out)); +process.exit(0); +`; + +interface DriverStep { + userId?: unknown; + displayName?: unknown; + plan?: unknown; + error?: unknown; + threw?: unknown; +} + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + // Unique suffix keeps signup emails collision-free across attempts. + const suffix = Date.now().toString(36); + const email = `alex-${suffix}@example.com`; + const password = 'correct-horse-battery'; + const wrongPassword = 'wrong-horse-battery'; + const displayName = 'Alex Doe'; + + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const publishableKey = str(status.PUBLISHABLE_KEY); + if (!apiUrl || !publishableKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/PUBLISHABLE_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + const written = await writeDriver(ctx); + if (!written.ok) { + return fail( + 'installed the eval driver', + written.stderr.trim() || written.stdout.trim() + ); + } + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_PUBLISHABLE_KEY="${publishableKey}" ` + + `node ${DRIVER} "${email}" "${password}" "${displayName}" "${wrongPassword}"`, + { timeoutMs: 60_000 } + ); + const out = parseDriverOutput(run.stdout); + checks.push({ + name: 'auth module loads and the driver completes', + passed: out !== undefined, + notes: out + ? 'driver produced a result' + : `no driver output — ${preview(run.stderr || run.stdout)}`, + }); + + const signUp = (out?.signUp ?? {}) as DriverStep; + const signInWrong = (out?.signInWrong ?? {}) as DriverStep; + const signIn = (out?.signIn ?? {}) as DriverStep; + const profile = (out?.profile ?? {}) as DriverStep; + + // Ground truth from the database (superuser query bypasses RLS). + const { rows: userRows } = await ctx.query( + `select u.id::text as id, p.display_name, p.plan + from auth.users u + left join public.profiles p on p.id = u.id + where u.email = '${email}'` + ); + const dbUser = userRows[0]; + + checks.push({ + name: 'signUp creates the account and returns its user id', + passed: !!dbUser && !!signUp.userId && signUp.userId === dbUser.id, + notes: dbUser + ? `db user ${dbUser.id}, signUp returned ${JSON.stringify(signUp)}` + : 'no auth.users row for the signup email', + }); + + // The seeded trigger falls back to the email local part, so the real + // display name only arrives if signUp sent it as user metadata. + checks.push({ + name: 'signup metadata reaches the profile (display name)', + passed: dbUser?.display_name === displayName, + notes: `profiles.display_name = ${JSON.stringify(dbUser?.display_name ?? null)}`, + }); + + checks.push({ + name: 'wrong password is rejected gracefully (no throw, no session)', + passed: !!signInWrong.error && !signInWrong.userId && !signInWrong.threw, + notes: JSON.stringify(signInWrong), + }); + + checks.push({ + name: 'signIn with the right password returns the user id', + passed: !!dbUser && signIn.userId === dbUser.id, + notes: JSON.stringify(signIn), + }); + + checks.push({ + name: "getMyProfile returns the signed-in user's profile", + passed: profile.displayName === displayName && profile.plan === 'free', + notes: JSON.stringify(profile), + }); + + // Client-side code: RLS + the publishable key are enough; the secret / + // service-role key must not appear anywhere in the app. + const secretScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules 'sb_secret_|SERVICE_ROLE' ${APP_DIR} || true` + ); + checks.push({ + name: 'app code does not use the secret / service-role key', + passed: secretScan.stdout.trim() === '', + notes: secretScan.stdout.trim() || 'no secret-key references found', + }); + + // GATING: the auth layer must be built on supabase-js, even though the + // prompt never names it. + checks.push(await sdkUsageCheck(ctx)); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function writeDriver(ctx: LocalStackEvalContext) { + const encoded = Buffer.from(DRIVER_SOURCE, 'utf-8').toString('base64'); + return ctx.exec(`echo ${encoded} | base64 -d > ${APP_DIR}/${DRIVER}`); +} + +function parseDriverOutput( + stdout: string +): Record | undefined { + const line = stdout + .split('\n') + .find((candidate) => candidate.includes(DRIVER_MARKER)); + if (!line) return undefined; + try { + return JSON.parse( + line.slice(line.indexOf(DRIVER_MARKER) + DRIVER_MARKER.length) + ); + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-auth-001-email-password-flow/PROMPT.md b/evals/build-auth-001-email-password-flow/PROMPT.md new file mode 100644 index 00000000..9061dada --- /dev/null +++ b/evals/build-auth-001-email-password-flow/PROMPT.md @@ -0,0 +1,31 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - auth + - database +topic: + - sdk + - rls +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + The signup → profile-trigger flow is a recurring pain point + (supabase/supabase#37497, supabase/supabase#35997, and the canonical + pattern in https://supabase.com/docs/guides/auth/managing-user-data); + auth is also the largest supabase-js surface with no dedicated eval + coverage. +--- + +Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in +there describe what each function should do. People sign up with an email, +password, and display name, sign back in later, and the app greets them with +their profile. + +The Supabase project for this app is in `supabase/` and already running +locally. When you're done, the functions should work for real against it. diff --git a/evals/build-auth-001-email-password-flow/README.md b/evals/build-auth-001-email-password-flow/README.md new file mode 100644 index 00000000..5a238bf0 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/README.md @@ -0,0 +1,19 @@ +# build-auth-001-email-password-flow + +Benchmark for supabase-js auth flows. The prompt is a casual "our app needs +accounts" ask pointing at stubbed functions in `local/app/src/auth.mjs` +(signUp / signIn / getMyProfile) — it never names supabase-js; the +"uses @supabase/supabase-js" check is GATING, like its precedent in +build-functions-005. + +The seed teaches through data rather than the prompt: the `profiles` trigger +falls back to the email local part unless the signup sends `display_name` +as user metadata, so the "display name reaches the profile" check only +passes when the agent wires `signUp` with `options.data`. RLS on `profiles` +plus the publishable key make the client-side path the only sanctioned one; +a "no secret key in app code" check guards the boundary. + +Scoring installs a small driver (`app/eval-driver.mjs`) that imports the +agent's module and exercises the contract in one process — sign up, wrong +password (must not throw), correct sign-in, profile read — then verifies +results against the database with the superuser connection. diff --git a/evals/build-auth-001-email-password-flow/local/app/package.json b/evals/build-auth-001-email-password-flow/local/app/package.json new file mode 100644 index 00000000..ccbe7c7e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "acme-app", + "private": true, + "type": "module" +} diff --git a/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs new file mode 100644 index 00000000..2e312295 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs @@ -0,0 +1,36 @@ +// Auth layer for the app. The rest of the app calls these three functions; +// wire them up to our Supabase project (it's running locally — see +// ../../supabase). Connection settings come from the environment: +// SUPABASE_URL and SUPABASE_PUBLISHABLE_KEY. +// +// This module is used from client-side code, so it must only ever hold the +// publishable (client) key. + +/** + * Create an account with email + password. `displayName` should end up as + * the user's profile display name. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure. + */ +export async function signUp(email, password, displayName) { + throw new Error('TODO: implement signUp'); +} + +/** + * Sign in with email + password. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure + * (e.g. wrong password) — it must not throw for bad credentials. + */ +export async function signIn(email, password) { + throw new Error('TODO: implement signIn'); +} + +/** + * The currently signed-in user's profile from the `profiles` table, as + * `{ displayName, plan }`. Resolves to `{ error: string }` when nobody is + * signed in. + */ +export async function getMyProfile() { + throw new Error('TODO: implement getMyProfile'); +} diff --git a/evals/build-auth-001-email-password-flow/local/supabase/config.toml b/evals/build-auth-001-email-password-flow/local/supabase/config.toml new file mode 100644 index 00000000..275fb4b8 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-auth-flow" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql new file mode 100644 index 00000000..b0eec56e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql @@ -0,0 +1,53 @@ +-- Public profile for each account, created automatically on signup. Guarded +-- by row-level security so users can only see and edit their own profile. +create table public.profiles ( + id uuid primary key references auth.users (id) on delete cascade, + display_name text not null, + plan text not null default 'free', + created_at timestamptz not null default now() +); + +alter table public.profiles enable row level security; + +create policy "users can read their own profile" + on public.profiles + for select + to authenticated + using (auth.uid() = id); + +create policy "users can update their own profile" + on public.profiles + for update + to authenticated + using (auth.uid() = id) + with check (auth.uid() = id); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows. +grant select, update on public.profiles to authenticated; + +-- Create the profile row when a user signs up. The display name comes from +-- the signup user metadata; if the app doesn't send one, we fall back to the +-- email local part. +create function public.handle_new_user() +returns trigger +language plpgsql +security definer +set search_path = '' +as $$ +begin + insert into public.profiles (id, display_name) + values ( + new.id, + coalesce( + new.raw_user_meta_data ->> 'display_name', + split_part(new.email, '@', 1) + ) + ); + return new; +end; +$$; + +create trigger on_auth_user_created + after insert on auth.users + for each row execute function public.handle_new_user(); diff --git a/evals/build-dataapi-001-relational-report/EVAL.ts b/evals/build-dataapi-001-relational-report/EVAL.ts new file mode 100644 index 00000000..0454fb8c --- /dev/null +++ b/evals/build-dataapi-001-relational-report/EVAL.ts @@ -0,0 +1,214 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Relational-report benchmark for the Data API: the prompt asks the agent to +// finish a backend reporting script over a seeded relational schema +// (customers → orders → order_items → products) and never names supabase-js — +// the "uses @supabase/supabase-js" check is GATING, and shelling out to +// psql / a raw Postgres driver instead fails. Expected numbers are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'report.mjs'; + +interface ReportRow { + customer: unknown; + orderCount: unknown; + totalCents: unknown; + topProduct: unknown; +} + +const EXPECTED_SQL = ` +with per_product as ( + select c.name as customer, p.name as product, + sum(oi.quantity)::int as units + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name, p.name +), totals as ( + select c.name as customer, + count(distinct o.id)::int as order_count, + sum(oi.quantity * p.price_cents)::int as total_cents + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name +) +select t.customer, + t.order_count, + t.total_cents, + (select pp.product + from per_product pp + where pp.customer = t.customer + order by pp.units desc, pp.product asc + limit 1) as top_product + from totals t + order by t.customer asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + customer: row.customer, + orderCount: row.order_count, + totalCents: row.total_cents, + topProduct: row.top_product, + })); + const normalized = (actual ?? []).map((row) => ({ + customer: row.customer, + orderCount: row.orderCount, + totalCents: row.totalCents, + topProduct: row.topProduct, + })); + checks.push({ + name: 'report numbers match the database (per customer, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('customers').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): ReportRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as ReportRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-001-relational-report/PROMPT.md b/evals/build-dataapi-001-relational-report/PROMPT.md new file mode 100644 index 00000000..258b8ae5 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/PROMPT.md @@ -0,0 +1,27 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - kong + - postgrest +projectRunning: true +motivation: >- + Relationship embedding is the query-builder surface users trip on most + (supabase/postgrest-js#609, supabase/postgrest-js#611, + supabase/supabase-js#1639), and the most-used supabase-js surface had no + dedicated eval coverage. +--- + +We need the nightly sales report working. `app/report.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON summary of what +each customer has ordered. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right numbers. diff --git a/evals/build-dataapi-001-relational-report/local/app/package.json b/evals/build-dataapi-001-relational-report/local/app/package.json new file mode 100644 index 00000000..81652eb0 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "reporting-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-001-relational-report/local/app/report.mjs b/evals/build-dataapi-001-relational-report/local/app/report.mjs new file mode 100644 index 00000000..757366f8 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/report.mjs @@ -0,0 +1,22 @@ +// Nightly sales report, run inside our Node backend worker: +// +// node report.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per customer who has placed at +// least one order, sorted by customer name: +// +// { +// "customer": string, // customer name +// "orderCount": number, // how many orders they placed +// "totalCents": number, // total spent across all their orders +// "topProduct": string // product they bought the most units of +// } +// +// If two products tie on units, topProduct is the alphabetically first one. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-001-relational-report/local/supabase/config.toml b/evals/build-dataapi-001-relational-report/local/supabase/config.toml new file mode 100644 index 00000000..8ca57d9f --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-orders-report" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql new file mode 100644 index 00000000..29f889c1 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql @@ -0,0 +1,69 @@ +-- Order history for the reporting worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.customers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + price_cents int not null +); + +create table public.orders ( + id bigint generated always as identity primary key, + customer_id bigint not null references public.customers (id), + ordered_at timestamptz not null default now() +); + +create table public.order_items ( + id bigint generated always as identity primary key, + order_id bigint not null references public.orders (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity > 0) +); + +alter table public.customers enable row level security; +alter table public.products enable row level security; +alter table public.orders enable row level security; +alter table public.order_items enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.customers, public.products, public.orders, + public.order_items to service_role; + +-- Seed data. Alan has no orders and must not appear in the report. +insert into public.customers (name, email) values + ('Ada Lovelace', 'ada@example.com'), + ('Grace Hopper', 'grace@example.com'), + ('Linus Pauling', 'linus@example.com'), + ('Alan Turing', 'alan@example.com'); + +insert into public.products (name, price_cents) values + ('Keyboard', 4500), + ('Mouse', 2500), + ('Monitor', 32000), + ('Cable', 900); + +insert into public.orders (customer_id, ordered_at) values + (1, '2026-06-01T10:00:00Z'), + (1, '2026-06-14T09:30:00Z'), + (2, '2026-06-03T16:20:00Z'), + (2, '2026-06-20T11:05:00Z'), + (3, '2026-06-08T14:45:00Z'); + +insert into public.order_items (order_id, product_id, quantity) values + (1, 1, 2), -- Ada: 2x Keyboard + (1, 4, 1), -- Ada: 1x Cable + (2, 3, 1), -- Ada: 1x Monitor + (3, 2, 3), -- Grace: 3x Mouse + (4, 1, 1), -- Grace: 1x Keyboard + (4, 4, 4), -- Grace: 4x Cable + (5, 3, 2), -- Linus: 2x Monitor + (5, 2, 1); -- Linus: 1x Mouse diff --git a/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts new file mode 100644 index 00000000..9c212f57 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts @@ -0,0 +1,230 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Legacy → new API key migration (regression): the seeded app authenticates +// with the local stack's legacy demo JWTs (anon + service_role) and works +// out of the box; the task is to move it to the new sb_publishable_… / +// sb_secret_… keys without breaking it. Scored on behavior (both scripts +// still print the right data — the stats script's RLS-bypassing count only +// works with a genuine secret key), on the legacy JWTs being gone, and on +// the key boundary staying intact (the public script must not end up holding +// the secret key). + +const APP_DIR = 'app'; + +// Header+payload prefix shared by both legacy local demo JWTs +// ({"iss":"supabase-demo",…} signed with the default local JWT secret) — +// matching on it catches either key regardless of the signature bytes. +const LEGACY_JWT_MARKER = 'eyJpc3MiOiJzdXBhYmFzZS1kZW1vIi'; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const publishableKey = str(status.PUBLISHABLE_KEY); + const secretKey = str(status.SECRET_KEY); + if (!publishableKey || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the keys, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + // Ground truth from the seeded database. + const { rows: publishedRows } = await ctx.query( + `select title from public.posts where published order by title asc` + ); + const expectedTitles = publishedRows.map((row) => row.title); + const { rows: draftRows } = await ctx.query( + `select count(*)::int as n from public.posts where not published` + ); + const expectedDrafts = Number(draftRows[0]?.n ?? -1); + + // 1. The public script still lists published posts (client-key path). + const posts = await ctx.exec(`cd ${APP_DIR} && npm run -s posts`, { + timeoutMs: 60_000, + }); + const postTitles = parseJson(posts.stdout, '[', ']'); + checks.push({ + name: 'posts script still lists published posts', + passed: + JSON.stringify(postTitles ?? null) === JSON.stringify(expectedTitles), + notes: postTitles + ? `got ${JSON.stringify(postTitles)}` + : `no JSON output — ${preview(posts.stderr || posts.stdout)}`, + }); + + // 2. The internal script still counts drafts. Drafts are invisible to the + // publishable key (RLS), so a correct count proves a working secret key. + const stats = await ctx.exec(`cd ${APP_DIR} && npm run -s stats`, { + timeoutMs: 60_000, + }); + const statsOut = parseJson(stats.stdout, '{', '}') as + | { drafts?: unknown } + | undefined; + checks.push({ + name: 'stats script still counts drafts (secret key bypasses RLS)', + passed: statsOut?.drafts === expectedDrafts, + notes: statsOut + ? `got ${JSON.stringify(statsOut)}, expected ${expectedDrafts} drafts` + : `no JSON output — ${preview(stats.stderr || stats.stdout)}`, + }); + + // 3. The legacy JWTs are gone from the app (env files included). + const legacyScan = await ctx.exec( + `grep -rl --exclude-dir=node_modules '${LEGACY_JWT_MARKER}' ${APP_DIR} || true` + ); + checks.push({ + name: 'legacy anon/service_role JWTs removed from the app', + passed: legacyScan.stdout.trim() === '', + notes: + legacyScan.stdout.trim().replace(/\s+/g, ', ') || + 'no legacy JWTs found', + }); + + // 4. Key boundary intact: the public script must not hold the secret key, + // neither as a literal nor via an env var that resolves to it. + checks.push(await publicScriptKeyCheck(ctx)); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +/** + * The public (posts) script must not use the secret key: no sb_secret_ + * literal in its source, and no reference to an env var whose value in .env + * is a secret key. + */ +async function publicScriptKeyCheck( + ctx: LocalStackEvalContext +): Promise { + const NAME = 'public script does not hold the secret key'; + const entry = await resolveScriptEntry(ctx, 'posts'); + const source = await ctx + .readFile(`${APP_DIR}/${entry}`) + .catch(() => undefined); + if (source === undefined) { + return { + name: NAME, + passed: false, + notes: `could not read ${APP_DIR}/${entry} to inspect`, + }; + } + if (source.includes('sb_secret_')) { + return { + name: NAME, + passed: false, + notes: `${entry} contains an sb_secret_ literal`, + }; + } + const env = await ctx.readFile(`${APP_DIR}/.env`).catch(() => ''); + const secretVars = parseEnv(env) + .filter(([, value]) => value.startsWith('sb_secret_')) + .map(([key]) => key); + const leaked = secretVars.filter((name) => source.includes(name)); + return { + name: NAME, + passed: leaked.length === 0, + notes: leaked.length + ? `${entry} references secret-key env var(s): ${leaked.join(', ')}` + : `${entry} holds no secret-key reference`, + }; +} + +/** File the given npm script runs, e.g. `posts` → `posts.mjs`. */ +async function resolveScriptEntry( + ctx: LocalStackEvalContext, + script: string +): Promise { + const fallback = `${script}.mjs`; + try { + const pkg = JSON.parse(await ctx.readFile(`${APP_DIR}/package.json`)) as { + scripts?: Record; + }; + const command = pkg.scripts?.[script] ?? ''; + return command.match(/[\w./-]+\.(?:mjs|cjs|js|ts)/)?.[0] ?? fallback; + } catch { + return fallback; + } +} + +function parseEnv(content: string): Array<[string, string]> { + return content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .flatMap((line) => { + const eq = line.indexOf('='); + if (eq === -1) return []; + const value = line + .slice(eq + 1) + .trim() + .replace(/^['"]|['"]$/g, ''); + return [[line.slice(0, eq).trim(), value] as [string, string]]; + }); +} + +function parseJson( + stdout: string, + open: string, + close: string +): unknown | undefined { + const start = stdout.indexOf(open); + const end = stdout.lastIndexOf(close); + if (start === -1 || end <= start) return undefined; + try { + return JSON.parse(stdout.slice(start, end + 1)); + } catch { + return undefined; + } +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md new file mode 100644 index 00000000..bfbd584d --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md @@ -0,0 +1,26 @@ +--- +stage: resolve +suite: regression +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - auth +topic: + - sdk + - security +services: + - kong + - postgrest +projectRunning: true +motivation: https://github.com/orgs/supabase/discussions/29260 +--- + +Heads-up from the platform team: the legacy JWT-based API keys (`anon` / +`service_role`) are going away for our projects soon, in favor of the new +publishable/secret keys. The little blog tooling app in `app/` still uses the +legacy keys. + +Migrate it over. Both scripts need to keep working — `npm run posts` and +`npm run stats` (run them from `app/`). The local Supabase project in +`supabase/` is already running. diff --git a/evals/resolve-sdk-001-legacy-key-migration/README.md b/evals/resolve-sdk-001-legacy-key-migration/README.md new file mode 100644 index 00000000..c5bc32d5 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/README.md @@ -0,0 +1,24 @@ +# resolve-sdk-001-legacy-key-migration + +A working blog-tooling app (`local/app/`) authenticates with the **legacy +local demo JWTs** — the deterministic `anon` / `service_role` keys every +local stack issues when `supabase/config.toml` doesn't override the JWT +secret (payload `{"iss":"supabase-demo",…}` signed with the default +`super-secret-jwt-token-with-at-least-32-characters-long`). That's what lets +the seed hardcode valid keys in `.env` before the stack exists. + +The task is to migrate the app to the new `sb_publishable_…` / `sb_secret_…` +keys (see the motivation link). The scorer checks behavior, not process: + +1. `npm run posts` still prints the published titles (client-key path). +2. `npm run stats` still prints the draft count — drafts are hidden from the + publishable key by RLS, so a correct count proves a real secret key. +3. No legacy demo JWT remains anywhere in `app/` (matched on the shared + header+payload prefix, so it catches both keys). +4. The public script doesn't end up holding the secret key, directly or via + an env var that resolves to one. + +Assumptions to keep in mind: the pinned `cliVersion` must expose +`PUBLISHABLE_KEY` / `SECRET_KEY` in `supabase status -o json`, and the stack +must still accept legacy JWTs by default (true as of 2.109.1) or the seeded +app would be broken before the agent starts. diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/.env b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env new file mode 100644 index 00000000..e9c51c59 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env @@ -0,0 +1,4 @@ +# Supabase keys for local dev (from `supabase status`) +SUPABASE_URL=http://127.0.0.1:54321 +SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0 +SUPABASE_SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json new file mode 100644 index 00000000..5cab9c72 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json @@ -0,0 +1,12 @@ +{ + "name": "blog-tools", + "private": true, + "type": "module", + "scripts": { + "posts": "node --env-file=.env posts.mjs", + "stats": "node --env-file=.env stats.mjs" + }, + "dependencies": { + "@supabase/supabase-js": "^2.58.0" + } +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs new file mode 100644 index 00000000..749820f0 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Public site: lists published post titles. Runs with the project's public +// (client-side) API key, so RLS applies. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_ANON_KEY +); + +const { data, error } = await supabase + .from('posts') + .select('title') + .order('title'); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify(data.map((post) => post.title))); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs new file mode 100644 index 00000000..a7155ccd --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Internal tooling: counts unpublished drafts across all posts. Trusted +// backend only — runs with the project's server-side key, which bypasses RLS. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_SERVICE_ROLE_KEY +); + +const { count, error } = await supabase + .from('posts') + .select('*', { count: 'exact', head: true }) + .eq('published', false); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify({ drafts: count })); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml new file mode 100644 index 00000000..d9722d7e --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-key-migration" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql new file mode 100644 index 00000000..28e2c5b8 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql @@ -0,0 +1,27 @@ +-- Blog posts. Published posts are public (readable with the client key); +-- drafts are only reachable by trusted backend code that bypasses RLS. +create table public.posts ( + id bigint generated always as identity primary key, + title text not null, + published boolean not null default false +); + +alter table public.posts enable row level security; + +create policy "anyone can read published posts" + on public.posts + for select + to anon, authenticated + using (published); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows for client roles; +-- service_role (trusted backend) bypasses RLS but still needs the privilege. +grant select on public.posts to anon, authenticated, service_role; + +insert into public.posts (title, published) values + ('Announcing vector buckets', true), + ('Realtime broadcast tips', true), + ('Row level security explained', true), + ('DRAFT: pricing update', false), + ('DRAFT: roadmap 2027', false); From bdd6c35488eb7e94261ccb757e5949f1e7273544 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:05:57 +0000 Subject: [PATCH 2/8] chore: refresh eval results --- apps/web/src/data/eval-results.json | 1409 ++++++++++++++++- .../web/src/data/regression-eval-results.json | 449 ++++++ 2 files changed, 1783 insertions(+), 75 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 5b28a57e..71542584 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -8,6 +8,88 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user ce86b374-4640-4173-9ae8-288753bb9c0a, signUp returned {\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8/build-auth-001-email-password-flow.json" + }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ @@ -304,6 +386,52 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8/build-dataapi-001-relational-report.json" + }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ @@ -2255,6 +2383,83 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user b4482103-7a65-4792-90f8-4047b254bdb5, signUp returned {\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8-no-skills/build-auth-001-email-password-flow.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ @@ -2480,6 +2685,47 @@ "modelId": "claude-opus-5", "reasoningEffort": "high" }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ @@ -3671,6 +3917,88 @@ "attempts": 1, "sourcePath": "claude-code-opus-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user f53f4ba0-315a-4668-a0a6-d49d98f06d9f, signUp returned {\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -3935,18 +4263,64 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, - "checks": [ + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ { "name": "all 3 tables exist (teams, members, tasks)", "passed": true @@ -5699,6 +6073,83 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 759339ee-2894-4cbe-a0ce-eda031d62308, signUp returned {\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -5870,6 +6321,47 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -6910,6 +7402,117 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user a346f0da-0630-4dfc-8d51-dfea9a715a98, signUp returned {\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js auth signUp signInWithPassword getUser profiles table user_metadata app_metadata\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + } + ], + "resultChars": 79170 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -7219,37 +7822,24 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true - }, - { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true - }, - { - "name": "foreign key constraints survived the restore", - "passed": true - }, - { - "name": "tasks_team_status_idx index survived the restore", - "passed": true - }, - { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" } ], "skills": { @@ -7264,10 +7854,10 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -7278,10 +7868,69 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "edge-functions", + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", "auth", "database" ], @@ -9501,6 +10150,83 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 32b0bd8e-2a03-4033-aceb-8723d458dac3, signUp returned {\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -9708,6 +10434,47 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -10967,50 +11734,60 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-auth-001-email-password-flow", "stage": "build", "product": [ - "database", - "data-api" + "auth", + "database" ], "topic": [ - "migrations", + "sdk", "rls" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" }, { - "name": "todos table is created by a migration file", - "passed": true + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user c651e731-ee81-4768-b3ae-4462c3fb2a9e, signUp returned {\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "signup metadata reaches the profile (display name)", "passed": true, - "notes": "found 2 rows" + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "getMyProfile returns the signed-in user's profile", "passed": true, - "notes": "0 rows" + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "app code does not use the secret / service-role key", "passed": true, - "notes": "2 rows" + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -11026,38 +11803,86 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Row Level Security policy authenticated role Data API grants select anon authenticated local development migrations seed\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js createClient signUp email password options data user metadata signInWithPassword getUser select single profile\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/auth/auth-anonymous", - "title": "Anonymous Sign-Ins" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" } ], - "resultChars": 100768 + "resultChars": 108618 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + } + ], + "resultChars": 52081 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript getUser current user getSession select single maybeSingle profiles RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", + "title": "Why is my select returning an empty data array and I have data in the table?" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + } + ], + "resultChars": 16223 } ] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" }, { "experiment": "codex-gpt-5.6", @@ -11066,31 +11891,132 @@ "agent": "codex", "modelProvider": "openai", "modelId": "gpt-5.6-sol", - "reasoningEffort": "medium" + "reasoningEffort": "low" }, - "eval": "build-cli-002-declarative-schema", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "database" + "database", + "data-api" ], "topic": [ - "declarative-schema", - "migrations" + "migrations", + "rls" ], "suite": "benchmark", "interface": "cli", "passed": true, "checks": [ { - "name": "supabase db diff used to generate the migration", - "passed": true - }, - { - "name": "schema file updated to include description column", + "name": "supabase project initialised (supabase/config.toml exists)", "passed": true }, { - "name": "a new migration was generated for the change", + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Row Level Security policy authenticated role Data API grants select anon authenticated local development migrations seed\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api", + "title": "Securing your API" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", + "title": "Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-anonymous", + "title": "Anonymous Sign-Ins" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + } + ], + "resultChars": 100768 + } + ] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-cli-002-declarative-schema", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "declarative-schema", + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "supabase db diff used to generate the migration", + "passed": true + }, + { + "name": "schema file updated to include description column", + "passed": true + }, + { + "name": "a new migration was generated for the change", "passed": true }, { @@ -11779,6 +12705,109 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient secret key backend select nested relationships pagination max rows range\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/guides/database/arrays", + "title": "Working With Arrays" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + } + ], + "resultChars": 26107 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"secret key backend apikey Authorization header sb_secret Supabase Data API\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 83375 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ @@ -13557,6 +14586,167 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 496d15e6-7d18-4029-9d81-03fea5521f76, signUp returned {\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select single profiles auth session\", limit: 8) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", + "title": "Login with LinkedIn" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", + "title": "Login with Slack" + } + ], + "resultChars": 156798 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data user_metadata signInWithPassword select maybeSingle single\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + }, + { + "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", + "title": "SignIn(email, password)" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + } + ], + "resultChars": 67379 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ @@ -14327,6 +15517,75 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase javascript client select nested relationships aggregate query node service role\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/schema" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 21571 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 97c715da..deec341f 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -622,6 +622,241 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"local development publishable secret API keys config.toml\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" + } + ], + "resultChars": 97856 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"local development config.toml auth.publishable_key auth.secret_key supabase start\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" + } + ], + "resultChars": 26442 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys publishable secret migration guide local development legacy JWT anon service_role\", limit: 5) { nodes { ... on Guide { title href } } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 749 + }, + { + "source": "web_fetch", + "query": "Extract all instructions relevant to: local development with the Supabase CLI, config.toml settings for publishable_key/secret_key, how to obtain or generate publishable/secret keys for a local project via `supabase status`, and how to update client code (supabase-js) from anon/service_role keys to the new keys.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" + } + ], + "resultChars": 1260 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"config.toml reference auth.publishable_key auth.secret_key local development default value\", limit: 5) { nodes { ... on Guide { title href } ... on CLICommandReference { title href } } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + } + ], + "resultChars": 615 + }, + { + "source": "web_fetch", + "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret in this config reference. Include the exact config key names, default values, and descriptions.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli/config.md" + } + ], + "resultChars": 215 + }, + { + "source": "web_fetch", + "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/managing-config.md" + } + ], + "resultChars": 669 + }, + { + "source": "web_fetch", + "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development. Include exact key names and default/example values.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cli/config.md" + } + ], + "resultChars": 215 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosted Kong publishable secret key apikey header REST API gateway configuration\", limit: 5) { nodes { ... on Guide { title href } ... on TroubleshootingGuide{title href} } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" + } + ], + "resultChars": 711 + }, + { + "source": "web_fetch", + "query": "Explain in detail how self-hosted / local Supabase (Kong, PostgREST, GoTrue) supports the new publishable and secret API keys instead of legacy anon/service_role JWTs. Include: config.toml or docker-compose/env var names needed, how Kong is configured to recognize sb_publishable_/sb_secret_ prefixed keys, whether these keys are still JWTs, and step-by-step setup instructions for local development with the Supabase CLI.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys.md" + } + ], + "resultChars": 2981 + } + ] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -1069,6 +1304,220 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable and secret API keys local development new API keys anon service_role migration\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + } + ], + "resultChars": 169611 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key local development config.toml\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", + "title": "Login with LinkedIn" + } + ], + "resultChars": 69263 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"JWT signing keys publishable secret API key migration anon service_role\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 61983 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"supabase CLI local development config.toml publishable secret key generate local\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", + "title": "Login with LinkedIn" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + } + ], + "resultChars": 78114 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"config.toml reference api.keys publishable_key secret_key local config\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" + } + ], + "resultChars": 21802 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"CLI config reference auth.publishable_key auth.secret_key\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 42641 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"CLI config.toml reference auth section publishable_key secret_key jwt_secret fields list\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + } + ], + "resultChars": 14597 + }, + { + "source": "web_fetch", + "query": "Find and quote all config fields under the [auth] section related to publishable_key, secret_key, jwt_secret, and any keys / API keys configuration. Also mention default values and whether they are required.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cli/config" + } + ], + "resultChars": 1424 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"auth.publishable_key auth.secret_key config reference default sb_publishable sb_secret local\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + } + ], + "resultChars": 78840 + } + ] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", From ae9c7a10cc1b1e28d132dff7daedf59279bd21b3 Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Tue, 28 Jul 2026 16:48:03 +0300 Subject: [PATCH 3/8] fix: add gotrue in services --- evals/build-dataapi-001-relational-report/PROMPT.md | 1 + .../local/supabase/.branches/_current_branch | 1 + .../local/supabase/.temp/cli-latest | 1 + evals/resolve-sdk-001-legacy-key-migration/PROMPT.md | 1 + 4 files changed, 4 insertions(+) create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest diff --git a/evals/build-dataapi-001-relational-report/PROMPT.md b/evals/build-dataapi-001-relational-report/PROMPT.md index 258b8ae5..00957811 100644 --- a/evals/build-dataapi-001-relational-report/PROMPT.md +++ b/evals/build-dataapi-001-relational-report/PROMPT.md @@ -9,6 +9,7 @@ product: topic: - sdk services: + - gotrue - kong - postgrest projectRunning: true diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch new file mode 100644 index 00000000..88d050b1 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest new file mode 100644 index 00000000..e9acfb34 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest @@ -0,0 +1 @@ +v2.110.0 \ No newline at end of file diff --git a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md index bfbd584d..7c59bf8c 100644 --- a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md +++ b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md @@ -10,6 +10,7 @@ topic: - sdk - security services: + - gotrue - kong - postgrest projectRunning: true From 2a46bb5a187696b6aa0ac87940582dd4e9db9185 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:20:20 +0000 Subject: [PATCH 4/8] chore: refresh eval results --- apps/web/src/data/eval-results.json | 532 +++++++++++------- .../web/src/data/regression-eval-results.json | 414 ++------------ 2 files changed, 381 insertions(+), 565 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 71542584..9e8b21bb 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user ce86b374-4640-4173-9ae8-288753bb9c0a, signUp returned {\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + "notes": "db user 80463b00-0201-45cb-96b2-3a8e3770f53e, signUp returned {\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + "notes": "{\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -398,12 +398,32 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -420,7 +440,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-opus-4.8/build-dataapi-001-relational-report.json" }, { @@ -2406,7 +2426,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user b4482103-7a65-4792-90f8-4047b254bdb5, signUp returned {\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + "notes": "db user 70a2faf6-f72e-4bef-a794-3a533d0b4dfe, signUp returned {\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -2421,7 +2441,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + "notes": "{\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -2700,9 +2720,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -3949,7 +3989,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user f53f4ba0-315a-4668-a0a6-d49d98f06d9f, signUp returned {\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + "notes": "db user 165887ac-a0d3-4589-bbb6-4efb98e21b93, signUp returned {\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -3964,7 +4004,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + "notes": "{\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -4278,9 +4318,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -4288,9 +4348,7 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] @@ -6105,7 +6163,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 759339ee-2894-4cbe-a0ce-eda031d62308, signUp returned {\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + "notes": "db user 49d31d27-be93-4d06-970f-c82f9746a729, signUp returned {\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6120,7 +6178,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + "notes": "{\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6342,12 +6400,32 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -6359,7 +6437,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" }, { @@ -7434,7 +7512,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user a346f0da-0630-4dfc-8d51-dfea9a715a98, signUp returned {\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + "notes": "db user de3c890b-dd5c-45c6-86da-1e8c94840211, signUp returned {\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -7449,7 +7527,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + "notes": "{\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -7480,13 +7558,9 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js auth signUp signInWithPassword getUser profiles table user_metadata app_metadata\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query {\n searchDocs(query: \"supabase-js signUp options data email password user metadata getUser getSession\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" @@ -7496,15 +7570,66 @@ "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 67792 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"JavaScript auth signInWithPassword getSession createClient supabase-js\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", + "title": "Login with Zoom" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", + "title": "Login with Slack" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", + "title": "Login with X / Twitter" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", + "title": "Login with GitHub" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", + "title": "Login with GitLab" } ], - "resultChars": 79170 + "resultChars": 220111 } ] }, @@ -7837,9 +7962,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -7852,7 +7997,35 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient service_role key select query\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/sveltekit", + "title": "Use Supabase with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 112740 + } + ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", @@ -10182,7 +10355,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 32b0bd8e-2a03-4033-aceb-8723d458dac3, signUp returned {\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + "notes": "db user b4fe3ed0-5c6b-46c1-a72d-e5e85b482175, signUp returned {\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -10197,7 +10370,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + "notes": "{\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -10458,9 +10631,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -11757,7 +11950,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user c651e731-ee81-4768-b3ae-4462c3fb2a9e, signUp returned {\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" + "notes": "db user 68bdb09b-5f95-4802-83ff-0c2fe62246ae, signUp returned {\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11772,7 +11965,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" + "notes": "{\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11803,21 +11996,13 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp email password options data user metadata signInWithPassword getUser select single profile\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser select profile single\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, { "url": "https://supabase.com/docs/guides/auth/passwords", "title": "Password-based Auth" @@ -11827,55 +12012,23 @@ "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - } - ], - "resultChars": 108618 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - } - ], - "resultChars": 52081 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript getUser current user getSession select single maybeSingle profiles RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/auth/enterprise-sso", + "title": "Enterprise Single Sign-On" } ], - "resultChars": 16223 + "resultChars": 109704 } ] }, @@ -12717,12 +12870,32 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -12738,65 +12911,64 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient secret key backend select nested relationships pagination max rows range\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js Node createClient select nested relationships service role local project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { - "url": "https://supabase.com/docs/guides/database/arrays", - "title": "Working With Arrays" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" } ], - "resultChars": 26107 + "resultChars": 66508 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"secret key backend apikey Authorization header sb_secret Supabase Data API\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"secret key sb_secret Authorization header apikey Supabase Data API server-side\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" } ], - "resultChars": 83375 + "resultChars": 90375 } ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" }, { @@ -14609,7 +14781,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 496d15e6-7d18-4029-9d81-03fea5521f76, signUp returned {\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + "notes": "db user 1675ab0f-f9ed-4305-acf2-88506d38a1bf, signUp returned {\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -14624,7 +14796,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + "notes": "{\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -14650,86 +14822,39 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select single profiles auth session\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select profile row level security authenticated user\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" - } - ], - "resultChars": 156798 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data user_metadata signInWithPassword select maybeSingle single\", limit: 10) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", + "title": "Build a User Management App with Ionic React" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", + "title": "Build a User Management App with Ionic Vue" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" } ], - "resultChars": 67379 + "resultChars": 97867 } ] }, @@ -15532,9 +15657,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -15545,30 +15690,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript client select nested relationships aggregate query node service role\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js select nested relationships aggregate querying foreign tables JavaScript\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/schema" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" }, { "url": "https://supabase.com/docs/guides/api/sql-to-api", "title": "Converting SQL to JavaScript API" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", + "title": "How to do automatic retries with `supabase-js`" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" } ], - "resultChars": 21571 + "resultChars": 42632 } ] }, diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index deec341f..009f73eb 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -644,12 +644,27 @@ "suite": "regression", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" } ], "skills": { @@ -662,199 +677,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"local development publishable secret API keys config.toml\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - } - ], - "resultChars": 97856 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"local development config.toml auth.publishable_key auth.secret_key supabase start\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - } - ], - "resultChars": 26442 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret migration guide local development legacy JWT anon service_role\", limit: 5) { nodes { ... on Guide { title href } } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 749 - }, - { - "source": "web_fetch", - "query": "Extract all instructions relevant to: local development with the Supabase CLI, config.toml settings for publishable_key/secret_key, how to obtain or generate publishable/secret keys for a local project via `supabase status`, and how to update client code (supabase-js) from anon/service_role keys to the new keys.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" - } - ], - "resultChars": 1260 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"config.toml reference auth.publishable_key auth.secret_key local development default value\", limit: 5) { nodes { ... on Guide { title href } ... on CLICommandReference { title href } } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 615 - }, - { - "source": "web_fetch", - "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret in this config reference. Include the exact config key names, default values, and descriptions.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/config.md" - } - ], - "resultChars": 215 - }, - { - "source": "web_fetch", - "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/managing-config.md" - } - ], - "resultChars": 669 - }, - { - "source": "web_fetch", - "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development. Include exact key names and default/example values.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cli/config.md" - } - ], - "resultChars": 215 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"self-hosted Kong publishable secret key apikey header REST API gateway configuration\", limit: 5) { nodes { ... on Guide { title href } ... on TroubleshootingGuide{title href} } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" - } - ], - "resultChars": 711 - }, - { - "source": "web_fetch", - "query": "Explain in detail how self-hosted / local Supabase (Kong, PostgREST, GoTrue) supports the new publishable and secret API keys instead of legacy anon/service_role JWTs. Include: config.toml or docker-compose/env var names needed, how Kong is configured to recognize sb_publishable_/sb_secret_ prefixed keys, whether these keys are still JWTs, and step-by-step setup instructions for local development with the Supabase CLI.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys.md" - } - ], - "resultChars": 2981 - } - ] + "calls": [] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" }, { @@ -1326,12 +1153,27 @@ "suite": "regression", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" } ], "skills": { @@ -1339,183 +1181,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable and secret API keys local development new API keys anon service_role migration\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - } - ], - "resultChars": 169611 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key local development config.toml\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - } - ], - "resultChars": 69263 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JWT signing keys publishable secret API key migration anon service_role\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 61983 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"supabase CLI local development config.toml publishable secret key generate local\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - } - ], - "resultChars": 78114 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"config.toml reference api.keys publishable_key secret_key local config\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - } - ], - "resultChars": 21802 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"CLI config reference auth.publishable_key auth.secret_key\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 42641 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"CLI config.toml reference auth section publishable_key secret_key jwt_secret fields list\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - } - ], - "resultChars": 14597 - }, - { - "source": "web_fetch", - "query": "Find and quote all config fields under the [auth] section related to publishable_key, secret_key, jwt_secret, and any keys / API keys configuration. Also mention default values and whether they are required.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cli/config" - } - ], - "resultChars": 1424 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"auth.publishable_key auth.secret_key config reference default sb_publishable sb_secret local\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - } - ], - "resultChars": 78840 - } - ] + "calls": [] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" }, { From 99583132fe02c2d7cf3234d3606abd7f8f9e57e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:31:25 +0000 Subject: [PATCH 5/8] chore: refresh eval results --- apps/web/src/data/eval-results.json | 481 ++++++++++++------ .../web/src/data/regression-eval-results.json | 31 +- 2 files changed, 356 insertions(+), 156 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 9e8b21bb..d7818f53 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 80463b00-0201-45cb-96b2-3a8e3770f53e, signUp returned {\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" + "notes": "db user ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5, signUp returned {\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" + "notes": "{\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -2426,7 +2426,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 70a2faf6-f72e-4bef-a794-3a533d0b4dfe, signUp returned {\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" + "notes": "db user db370024-8f1e-4509-836e-53f273ac466c, signUp returned {\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -2441,7 +2441,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" + "notes": "{\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -2717,7 +2717,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -2736,8 +2736,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/report.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -2754,7 +2754,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-001-relational-report.json" }, { @@ -3989,7 +3989,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 165887ac-a0d3-4589-bbb6-4efb98e21b93, signUp returned {\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" + "notes": "db user d85c49f2-9854-4de0-9395-9b4048713489, signUp returned {\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -4004,7 +4004,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" + "notes": "{\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -4315,7 +4315,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -4334,8 +4334,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/report.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -4355,7 +4355,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" }, { @@ -6163,7 +6163,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 49d31d27-be93-4d06-970f-c82f9746a729, signUp returned {\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" + "notes": "db user 175ffe86-cecb-4504-b9b2-843114d08b2c, signUp returned {\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6178,7 +6178,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" + "notes": "{\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -7502,7 +7502,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -7512,7 +7512,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user de3c890b-dd5c-45c6-86da-1e8c94840211, signUp returned {\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" + "notes": "db user 3251eadd-3f5a-452a-9085-d7e690fac0df, signUp returned {\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -7527,7 +7527,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" + "notes": "{\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -7541,8 +7541,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -7556,80 +7556,103 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog.md breaking-change auth supabase-js", + "pages": [] + }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp options data email password user metadata getUser getSession\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession createClient auth local storage browser client\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", + "title": "Login with GitHub" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", + "title": "Login with X / Twitter" } ], - "resultChars": 67792 + "resultChars": 74654 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"JavaScript auth signInWithPassword getSession createClient supabase-js\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query {\n searchDocs(query: \"supabase-js auth.signUp signInWithPassword getUser select profile from table reference javascript\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, { "url": "https://supabase.com/docs/guides/auth/passwords", "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", - "title": "Login with Zoom" + "url": "https://supabase.com/docs/reference/dart/auth-signup" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", - "title": "Login with X / Twitter" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + } + ], + "resultChars": 163499 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"reference javascript auth signInWithPassword\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", - "title": "Login with GitHub" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" } ], - "resultChars": 220111 + "resultChars": 3390 } ] }, @@ -7997,35 +8020,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient service_role key select query\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/sveltekit", - "title": "Use Supabase with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 112740 - } - ] + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", @@ -10355,7 +10350,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user b4fe3ed0-5c6b-46c1-a72d-e5e85b482175, signUp returned {\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" + "notes": "db user 40f508e4-cd55-4379-9f5a-3c9392f94902, signUp returned {\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -10370,7 +10365,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" + "notes": "{\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -10393,7 +10388,157 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"auth sign up with metadata password grant profiles row level security local supabase auth\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/enterprise-sso", + "title": "Enterprise Single Sign-On" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" + } + ], + "resultChars": 117348 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"auth signup options data signInWithPassword access token refresh token user endpoint REST\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions/implicit-flow", + "title": "Implicit flow" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + } + ], + "resultChars": 104157 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"JavaScript reference get user current session auth user endpoint\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + }, + { + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/egress", + "title": "Manage Egress usage" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-currentuser" + } + ], + "resultChars": 130065 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"refresh token grant_type refresh_token supabase auth v1 token json body\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login", + "title": "Social Login" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", + "title": "Advanced guide" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 86530 + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -11950,7 +12095,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 68bdb09b-5f95-4802-83ff-0c2fe62246ae, signUp returned {\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" + "notes": "db user 58379f63-4af9-429b-9ea2-ba595cbb73a8, signUp returned {\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11965,7 +12110,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" + "notes": "{\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11996,39 +12141,72 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser select profile single\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"supabase javascript auth signUp options data user metadata signInWithPassword getUser select profile RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 55094 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript createClient signUp signInWithPassword select single auth session browser persistSession current\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" }, { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", + "title": "Login with X / Twitter" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", + "title": "Login with Zoom" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-kakao", + "title": "Login with Kakao" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-initialize" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" } ], - "resultChars": 109704 + "resultChars": 93981 } ] }, @@ -12870,7 +13048,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -12889,8 +13067,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -12911,64 +13089,65 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js Node createClient select nested relationships service role local project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase javascript select nested relationships aggregate count sum foreign tables\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", + "title": "How to do automatic retries with `supabase-js`" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" } ], - "resultChars": 66508 + "resultChars": 42632 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"secret key sb_secret Authorization header apikey Supabase Data API server-side\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"secret key backend Data API apikey Authorization header Supabase REST\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" } ], - "resultChars": 90375 + "resultChars": 65167 } ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" }, { @@ -14781,7 +14960,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 1675ab0f-f9ed-4305-acf2-88506d38a1bf, signUp returned {\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" + "notes": "db user b72ca89c-acb9-48dc-9a16-80af3c5b09ca, signUp returned {\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -14796,7 +14975,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" + "notes": "{\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -14822,39 +15001,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select profile row level security authenticated user\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"supabase-js signUp user metadata signInWithPassword auth getUser select profiles RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", - "title": "Build a User Management App with Ionic Vue" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" } ], - "resultChars": 97867 + "resultChars": 53768 } ] }, @@ -15690,31 +15861,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js select nested relationships aggregate querying foreign tables JavaScript\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase REST API JavaScript fetch apikey Authorization service role secret key Range header pagination PostgREST\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", - "title": "How to do automatic retries with `supabase-js`" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/api/handling-errors-in-supabase-js", + "title": "Handling errors in `supabase-js`" } ], - "resultChars": 42632 + "resultChars": 39367 } ] }, diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 009f73eb..32fc1535 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -677,7 +677,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable and secret API keys migration from anon and service_role\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + } + ], + "resultChars": 107177 + } + ] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", From 1a06c3dde64a22d5769243285b6caf5311080ce4 Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Tue, 28 Jul 2026 18:10:32 +0300 Subject: [PATCH 6/8] feat: one more eval --- .../EVAL.ts | 202 ++++++++++++++++++ .../PROMPT.md | 32 +++ .../README.md | 6 + .../local/app/package.json | 5 + .../local/app/restock.mjs | 24 +++ .../local/supabase/.branches/_current_branch | 1 + .../local/supabase/.temp/cli-latest | 1 + .../local/supabase/config.toml | 165 ++++++++++++++ .../migrations/0000_inventory_schema.sql | 68 ++++++ 9 files changed, 504 insertions(+) create mode 100644 evals/build-dataapi-002-restock-alert-report/EVAL.ts create mode 100644 evals/build-dataapi-002-restock-alert-report/PROMPT.md create mode 100644 evals/build-dataapi-002-restock-alert-report/README.md create mode 100644 evals/build-dataapi-002-restock-alert-report/local/app/package.json create mode 100644 evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql diff --git a/evals/build-dataapi-002-restock-alert-report/EVAL.ts b/evals/build-dataapi-002-restock-alert-report/EVAL.ts new file mode 100644 index 00000000..948e6271 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/EVAL.ts @@ -0,0 +1,202 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Companion to build-dataapi-001-relational-report: same "unnamed SDK, empty +// package.json, backend worker script" shape, different schema (warehouses → +// inventory → products → suppliers) and aggregation (below-threshold restock +// alert, not a sales rollup). Checks whether the SDK-adoption split found +// there (claude-code 4/4, codex 0/4) generalizes or was specific to that +// prompt. The "uses @supabase/supabase-js" check is GATING, and shelling out +// to psql / a raw Postgres driver instead fails. Expected alerts are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'restock.mjs'; + +interface AlertRow { + warehouse: unknown; + product: unknown; + quantity: unknown; + reorderThreshold: unknown; + supplierEmail: unknown; +} + +const EXPECTED_SQL = ` +select w.name as warehouse, + p.name as product, + i.quantity, + p.reorder_threshold, + s.email as supplier_email + from public.inventory i + join public.warehouses w on w.id = i.warehouse_id + join public.products p on p.id = i.product_id + join public.suppliers s on s.id = p.supplier_id + where i.quantity < p.reorder_threshold + order by w.name asc, p.name asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorder_threshold, + supplierEmail: row.supplier_email, + })); + const normalized = (actual ?? []).map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorderThreshold, + supplierEmail: row.supplierEmail, + })); + checks.push({ + name: 'alerts match the database (below threshold, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('inventory').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): AlertRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as AlertRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-002-restock-alert-report/PROMPT.md b/evals/build-dataapi-002-restock-alert-report/PROMPT.md new file mode 100644 index 00000000..5b3d1a90 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/PROMPT.md @@ -0,0 +1,32 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + build-dataapi-001-relational-report found every codex variant (0/4) skips + @supabase/supabase-js for a bare backend Data API script, hand-rolling raw + HTTP instead, while every claude-code variant (4/4) reached for it + unprompted. A single scenario isn't enough to tell a real model tendency + from a one-off artifact of that prompt's specific shape — this companion + scenario keeps the same "unnamed SDK, empty package.json, backend worker + script" shape but swaps in an unrelated schema and aggregation (inventory + restock alerts vs. sales report) to check whether the pattern generalizes. +--- + +Purchasing needs a restock alert. `app/restock.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON list of what +needs reordering, with who to email about it. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right alerts. diff --git a/evals/build-dataapi-002-restock-alert-report/README.md b/evals/build-dataapi-002-restock-alert-report/README.md new file mode 100644 index 00000000..e9bb4ba6 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/README.md @@ -0,0 +1,6 @@ +Companion to `build-dataapi-001-relational-report`. Same shape (unnamed SDK, +empty `package.json`, bare backend worker script reading `SUPABASE_URL` / +`SUPABASE_SECRET_KEY` from the env) but a different schema and aggregation +(inventory restock alerts vs. a sales report), to check whether that eval's +SDK-adoption split (claude-code 4/4 vs. codex 0/4) is a real model tendency +or an artifact of that one prompt. diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/package.json b/evals/build-dataapi-002-restock-alert-report/local/app/package.json new file mode 100644 index 00000000..4f4a0bce --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "restock-alert-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs new file mode 100644 index 00000000..1d247baf --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs @@ -0,0 +1,24 @@ +// Restock alert worker, run inside our Node backend worker: +// +// node restock.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per warehouse/product +// combination that's below its reorder threshold, sorted by warehouse name +// then product name: +// +// { +// "warehouse": string, // warehouse name +// "product": string, // product name +// "quantity": number, // current quantity on hand +// "reorderThreshold": number, // reorder threshold for this product +// "supplierEmail": string // email of the product's supplier +// } +// +// Only include rows where quantity is strictly below the reorder threshold. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch new file mode 100644 index 00000000..88d050b1 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest new file mode 100644 index 00000000..e9acfb34 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest @@ -0,0 +1 @@ +v2.110.0 \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml new file mode 100644 index 00000000..06330f25 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-restock-alert" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql new file mode 100644 index 00000000..0dfdff73 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql @@ -0,0 +1,68 @@ +-- Inventory for the restock-alert worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.warehouses ( + id bigint generated always as identity primary key, + name text not null +); + +create table public.suppliers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + supplier_id bigint not null references public.suppliers (id), + reorder_threshold int not null +); + +create table public.inventory ( + id bigint generated always as identity primary key, + warehouse_id bigint not null references public.warehouses (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity >= 0) +); + +alter table public.warehouses enable row level security; +alter table public.suppliers enable row level security; +alter table public.products enable row level security; +alter table public.inventory enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.warehouses, public.suppliers, public.products, + public.inventory to service_role; + +-- Seed data. +insert into public.warehouses (name) values + ('North DC'), + ('South DC'), + ('West DC'); + +insert into public.suppliers (name, email) values + ('Acme Supplies', 'acme@example.com'), + ('Global Parts', 'parts@example.com'); + +insert into public.products (name, supplier_id, reorder_threshold) values + ('Widget', 1, 20), + ('Gadget', 2, 15), + ('Gizmo', 1, 10); + +-- warehouse_id, product_id, quantity. Below-threshold rows are the alerts; +-- West DC's Widget sits exactly at threshold and must NOT alert (strict +-- less-than), and West DC has no Gizmo row at all (untracked combos are +-- simply absent, not a zero-quantity alert). +insert into public.inventory (warehouse_id, product_id, quantity) values + (1, 1, 5), -- North DC / Widget: 5 < 20 -> alert + (1, 2, 30), -- North DC / Gadget: 30 >= 15 -> ok + (1, 3, 3), -- North DC / Gizmo: 3 < 10 -> alert + (2, 1, 25), -- South DC / Widget: 25 >= 20 -> ok + (2, 2, 2), -- South DC / Gadget: 2 < 15 -> alert + (2, 3, 12), -- South DC / Gizmo: 12 >= 10 -> ok + (3, 1, 20), -- West DC / Widget: 20 >= 20 -> ok (boundary, not strictly below) + (3, 2, 0); -- West DC / Gadget: 0 < 15 -> alert From 488a9857530f960556d28ad07c3a1eed3a1fa4d8 Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Wed, 5 Aug 2026 13:55:00 +0300 Subject: [PATCH 7/8] fix: address comments --- .gitignore | 6 +- .../EVAL.ts | 42 +++++++-- .../EVAL.ts | 41 ++++++--- .../local/supabase/.branches/_current_branch | 1 - .../local/supabase/.temp/cli-latest | 1 - .../EVAL.ts | 41 ++++++--- .../PROMPT.md | 5 ++ .../README.md | 5 +- .../local/supabase/.branches/_current_branch | 1 - .../local/supabase/.temp/cli-latest | 1 - .../EVAL.ts | 90 ++++++++++++++++++- 11 files changed, 196 insertions(+), 38 deletions(-) delete mode 100644 evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch delete mode 100644 evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest delete mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch delete mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest diff --git a/.gitignore b/.gitignore index 23951d46..5dc75e68 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,11 @@ dist/ .env .env*.local # eval seed data may include a .env with well-known local demo keys -!evals/*/local/**/.env +!evals/resolve-sdk-001-legacy-key-migration/local/app/.env + +# generated local CLI state, not eval seed data +evals/*/local/supabase/.temp/ +evals/*/local/supabase/.branches/ .DS_Store results/*/ .sync-tmp/ diff --git a/evals/build-auth-001-email-password-flow/EVAL.ts b/evals/build-auth-001-email-password-flow/EVAL.ts index f658d512..64b9c878 100644 --- a/evals/build-auth-001-email-password-flow/EVAL.ts +++ b/evals/build-auth-001-email-password-flow/EVAL.ts @@ -68,6 +68,19 @@ const scorer: LocalStackScorer = async (ctx) => { ); } + // Be generous about a missing install step; the eval is about auth, + // not npm. A no-op when the agent already installed dependencies. + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, + { timeoutMs: 180_000 } + ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } + const written = await writeDriver(ctx); if (!written.ok) { return fail( @@ -189,21 +202,34 @@ function parseDriverOutput( /** * GATING: some app code file must genuinely import @supabase/supabase-js — - * we match the quoted module specifier, not a bare mention in a comment. + * the specifier must be closed by a matching quote (so `-not-real` doesn't + * match) and sit on a `from`/`require(`/`import(` line that isn't commented + * out. Multi-line named imports still match: the closing `} from '…'` line + * always carries `from` alongside the specifier. */ async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { const NAME = 'implementation uses @supabase/supabase-js'; const scan = await ctx.exec( - `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + - `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + `grep -rnE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"(from|require\\(|import\\()\\s*['\\"](npm:)?@supabase/supabase-js['\\"]" ${APP_DIR} ` + + `| grep -vE ':[0-9]+:\\s*(//|\\*)' || true` ); - const files = scan.stdout.trim(); + const files = [ + ...new Set( + scan.stdout + .trim() + .split('\n') + .filter(Boolean) + .map((line) => line.slice(0, line.indexOf(':'))) + ), + ]; return { name: NAME, - passed: files !== '', - notes: files - ? `imports found in: ${files.replace(/\s+/g, ', ')}` - : 'no @supabase/supabase-js import found — this eval requires the SDK', + passed: files.length > 0, + notes: + files.length > 0 + ? `imports found in: ${files.join(', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', }; } diff --git a/evals/build-dataapi-001-relational-report/EVAL.ts b/evals/build-dataapi-001-relational-report/EVAL.ts index 0454fb8c..62ea25ca 100644 --- a/evals/build-dataapi-001-relational-report/EVAL.ts +++ b/evals/build-dataapi-001-relational-report/EVAL.ts @@ -68,10 +68,16 @@ const scorer: LocalStackScorer = async (ctx) => { // Be generous about a missing install step; the eval is about the report, // not npm. A no-op when the agent already installed dependencies. - await ctx.exec( - `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, { timeoutMs: 180_000 } ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } const run = await ctx.exec( `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, @@ -80,7 +86,7 @@ const scorer: LocalStackScorer = async (ctx) => { const actual = parseReport(run.stdout); checks.push({ name: 'report runs and prints JSON', - passed: actual !== undefined, + passed: run.ok && actual !== undefined, notes: actual !== undefined ? `exit ${run.exitCode}` @@ -165,21 +171,34 @@ function parseReport(stdout: string): ReportRow[] | undefined { /** * GATING: some app code file must genuinely import @supabase/supabase-js — - * we match the quoted module specifier, not a bare mention in a comment. + * the specifier must be closed by a matching quote (so `-not-real` doesn't + * match) and sit on a `from`/`require(`/`import(` line that isn't commented + * out. Multi-line named imports still match: the closing `} from '…'` line + * always carries `from` alongside the specifier. */ async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { const NAME = 'implementation uses @supabase/supabase-js'; const scan = await ctx.exec( - `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + - `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + `grep -rnE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"(from|require\\(|import\\()\\s*['\\"](npm:)?@supabase/supabase-js['\\"]" ${APP_DIR} ` + + `| grep -vE ':[0-9]+:\\s*(//|\\*)' || true` ); - const files = scan.stdout.trim(); + const files = [ + ...new Set( + scan.stdout + .trim() + .split('\n') + .filter(Boolean) + .map((line) => line.slice(0, line.indexOf(':'))) + ), + ]; return { name: NAME, - passed: files !== '', - notes: files - ? `imports found in: ${files.replace(/\s+/g, ', ')}` - : 'no @supabase/supabase-js import found — this eval requires the SDK', + passed: files.length > 0, + notes: + files.length > 0 + ? `imports found in: ${files.join(', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', }; } diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch deleted file mode 100644 index 88d050b1..00000000 --- a/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch +++ /dev/null @@ -1 +0,0 @@ -main \ No newline at end of file diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest deleted file mode 100644 index e9acfb34..00000000 --- a/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest +++ /dev/null @@ -1 +0,0 @@ -v2.110.0 \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/EVAL.ts b/evals/build-dataapi-002-restock-alert-report/EVAL.ts index 948e6271..352eede5 100644 --- a/evals/build-dataapi-002-restock-alert-report/EVAL.ts +++ b/evals/build-dataapi-002-restock-alert-report/EVAL.ts @@ -54,10 +54,16 @@ const scorer: LocalStackScorer = async (ctx) => { // Be generous about a missing install step; the eval is about the report, // not npm. A no-op when the agent already installed dependencies. - await ctx.exec( - `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, { timeoutMs: 180_000 } ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } const run = await ctx.exec( `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, @@ -66,7 +72,7 @@ const scorer: LocalStackScorer = async (ctx) => { const actual = parseReport(run.stdout); checks.push({ name: 'report runs and prints JSON', - passed: actual !== undefined, + passed: run.ok && actual !== undefined, notes: actual !== undefined ? `exit ${run.exitCode}` @@ -153,21 +159,34 @@ function parseReport(stdout: string): AlertRow[] | undefined { /** * GATING: some app code file must genuinely import @supabase/supabase-js — - * we match the quoted module specifier, not a bare mention in a comment. + * the specifier must be closed by a matching quote (so `-not-real` doesn't + * match) and sit on a `from`/`require(`/`import(` line that isn't commented + * out. Multi-line named imports still match: the closing `} from '…'` line + * always carries `from` alongside the specifier. */ async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { const NAME = 'implementation uses @supabase/supabase-js'; const scan = await ctx.exec( - `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + - `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + `grep -rnE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"(from|require\\(|import\\()\\s*['\\"](npm:)?@supabase/supabase-js['\\"]" ${APP_DIR} ` + + `| grep -vE ':[0-9]+:\\s*(//|\\*)' || true` ); - const files = scan.stdout.trim(); + const files = [ + ...new Set( + scan.stdout + .trim() + .split('\n') + .filter(Boolean) + .map((line) => line.slice(0, line.indexOf(':'))) + ), + ]; return { name: NAME, - passed: files !== '', - notes: files - ? `imports found in: ${files.replace(/\s+/g, ', ')}` - : 'no @supabase/supabase-js import found — this eval requires the SDK', + passed: files.length > 0, + notes: + files.length > 0 + ? `imports found in: ${files.join(', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', }; } diff --git a/evals/build-dataapi-002-restock-alert-report/PROMPT.md b/evals/build-dataapi-002-restock-alert-report/PROMPT.md index 5b3d1a90..9dcd9494 100644 --- a/evals/build-dataapi-002-restock-alert-report/PROMPT.md +++ b/evals/build-dataapi-002-restock-alert-report/PROMPT.md @@ -22,6 +22,11 @@ motivation: >- scenario keeps the same "unnamed SDK, empty package.json, backend worker script" shape but swaps in an unrelated schema and aggregation (inventory restock alerts vs. sales report) to check whether the pattern generalizes. + Agents defaulting to the wrong data-access path on a backend task is a + real, documented failure beyond this pair of evals too + (supabase/agent-skills#173: a Codex session reached for raw PostgREST + credentials and an admin browser instead of the intended tool for a data + read/update task). --- Purchasing needs a restock alert. `app/restock.mjs` has the spec in a diff --git a/evals/build-dataapi-002-restock-alert-report/README.md b/evals/build-dataapi-002-restock-alert-report/README.md index e9bb4ba6..47cc1eb4 100644 --- a/evals/build-dataapi-002-restock-alert-report/README.md +++ b/evals/build-dataapi-002-restock-alert-report/README.md @@ -3,4 +3,7 @@ empty `package.json`, bare backend worker script reading `SUPABASE_URL` / `SUPABASE_SECRET_KEY` from the env) but a different schema and aggregation (inventory restock alerts vs. a sales report), to check whether that eval's SDK-adoption split (claude-code 4/4 vs. codex 0/4) is a real model tendency -or an artifact of that one prompt. +or an artifact of that one prompt. Beyond this pair of evals, +supabase/agent-skills#173 documents the same underlying failure shape in the +wild — a Codex session reaching for raw PostgREST credentials and an admin +browser instead of the intended tool for a data task. diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch deleted file mode 100644 index 88d050b1..00000000 --- a/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch +++ /dev/null @@ -1 +0,0 @@ -main \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest deleted file mode 100644 index e9acfb34..00000000 --- a/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest +++ /dev/null @@ -1 +0,0 @@ -v2.110.0 \ No newline at end of file diff --git a/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts index 9c212f57..7921bd26 100644 --- a/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts +++ b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts @@ -1,5 +1,6 @@ import { type CheckResult, + type CommandResult, type LocalStackEvalContext, type LocalStackScorer, } from '@supabase-evals/core'; @@ -35,10 +36,16 @@ const scorer: LocalStackScorer = async (ctx) => { // Be generous about a missing install step; the eval is about the keys, // not npm. A no-op when the agent already installed dependencies. - await ctx.exec( - `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + const install = await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent`, { timeoutMs: 180_000 } ); + if (!install.ok) { + return fail( + 'installed app dependencies', + install.stderr.trim() || install.stdout.trim() + ); + } // Ground truth from the seeded database. const { rows: publishedRows } = await ctx.query( @@ -96,6 +103,18 @@ const scorer: LocalStackScorer = async (ctx) => { // neither as a literal nor via an env var that resolves to it. checks.push(await publicScriptKeyCheck(ctx)); + // 5. Prove both scripts actually depend on the keys, not just that their + // output happens to match: corrupt the live key value on disk and + // confirm the same script now breaks. Without this, a stub that already + // knows the expected titles/count — with no real Supabase call at all — + // would pass checks 1-4 for free. + checks.push( + await keyDependencyCheck(ctx, 'posts', publishableKey, 'publishable key') + ); + checks.push( + await keyDependencyCheck(ctx, 'stats', secretKey, 'secret key') + ); + return { passed: checks.every((c) => c.passed), checks }; } catch (error) { const msg = error instanceof Error ? error.message : String(error); @@ -151,6 +170,73 @@ async function publicScriptKeyCheck( }; } +/** + * Corrupts the on-disk copy of `keyValue` (in `.env`, or in the script's + * source if it's hardcoded there instead) and re-runs the given npm script, + * expecting it to break — proving the script reads the key at call time + * rather than being hardcoded or key-agnostic. Restores the original + * content afterward either way. + */ +async function keyDependencyCheck( + ctx: LocalStackEvalContext, + script: 'posts' | 'stats', + keyValue: string, + label: string +): Promise { + const NAME = `${script} script actually depends on its ${label}`; + const envPath = `${APP_DIR}/.env`; + const entry = await resolveScriptEntry(ctx, script); + const entryPath = `${APP_DIR}/${entry}`; + + const env = await ctx.readFile(envPath).catch(() => undefined); + const targetPath = env?.includes(keyValue) + ? envPath + : await ctx + .readFile(entryPath) + .then((source) => (source.includes(keyValue) ? entryPath : undefined)) + .catch(() => undefined); + if (targetPath === undefined) { + return { + name: NAME, + passed: false, + notes: `could not find the live ${label} on disk (checked ${envPath} and ${entryPath}) to corrupt`, + }; + } + + const original = await ctx.readFile(targetPath); + const corrupted = original.replaceAll( + keyValue, + `${keyValue}-corrupted-by-eval` + ); + await writeFile(ctx, targetPath, corrupted); + const run = await ctx + .exec(`cd ${APP_DIR} && npm run -s ${script}`, { timeoutMs: 60_000 }) + .finally(() => writeFile(ctx, targetPath, original)); + + const stillLooksValid = + script === 'posts' + ? parseJson(run.stdout, '[', ']') !== undefined + : (parseJson(run.stdout, '{', '}') as { drafts?: unknown } | undefined) + ?.drafts !== undefined; + return { + name: NAME, + passed: !run.ok || !stillLooksValid, + notes: + !run.ok || !stillLooksValid + ? `broke as expected with a corrupted ${label}` + : `still produced ${preview(run.stdout)} with a corrupted ${label} — looks hardcoded or key-agnostic`, + }; +} + +function writeFile( + ctx: LocalStackEvalContext, + path: string, + content: string +): Promise { + const encoded = Buffer.from(content, 'utf-8').toString('base64'); + return ctx.exec(`echo ${encoded} | base64 -d > ${path}`); +} + /** File the given npm script runs, e.g. `posts` → `posts.mjs`. */ async function resolveScriptEntry( ctx: LocalStackEvalContext, From 985f43cbb8d90ef132ed83e24faa867f4fcfde03 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 11:16:37 +0000 Subject: [PATCH 8/8] chore: refresh eval results --- apps/web/src/data/eval-results.json | 2345 ++++++++++++----- .../web/src/data/regression-eval-results.json | 59 +- 2 files changed, 1638 insertions(+), 766 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index d7818f53..2371d4c9 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1,67 +1,57 @@ [ { - "experiment": "claude-code-opus-5", + "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-5", + "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-auth-001-email-password-flow", + "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ - "auth", - "database" + "database", + "data-api" ], "topic": [ - "sdk", + "migrations", "rls" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5, signUp returned {\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true }, { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" + "name": "todos table is created by a migration file", + "passed": true }, { - "name": "wrong password is rejected gracefully (no throw, no session)", + "name": "todos table exists with at least 2 seeded rows", "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" + "notes": "found 2 rows" }, { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" + "name": "row level security is enabled on todos", + "passed": true }, { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + "name": "a SELECT policy targets the authenticated role", + "passed": true }, { - "name": "app code does not use the secret / service-role key", + "name": "REST API returns no todos to anonymous requests", "passed": true, - "notes": "no secret-key references found" + "notes": "error 42501: permission denied for table todos" }, { - "name": "implementation uses @supabase/supabase-js", + "name": "REST API returns the todos to authenticated requests", "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "notes": "2 rows" } ], "skills": { @@ -74,12 +64,35 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "web_fetch", + "query": "List any entries tagged breaking-change, and anything relevant to: local development with the CLI, database migrations, RLS policies, the anon/authenticated roles, API keys (publishable/secret vs anon/service_role), seed files, or config.toml. Include dates and links.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 8255 + }, + { + "source": "web_fetch", + "query": "Show the full recommended workflow and exact SQL for exposing a table in the public schema to the Data API, including GRANT statements for anon and authenticated roles, enabling RLS, and read-only access patterns. Quote the SQL verbatim.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/api/securing-your-api.md" + } + ], + "resultChars": 1245 + } + ] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-4.8/build-auth-001-email-password-flow.json" + "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" }, { "experiment": "claude-code-opus-4.8", @@ -90,6 +103,120 @@ "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restore pg_dump custom format dump into Supabase migrate existing postgres database\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", + "title": "Migrate from Neon to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", + "title": "Restore a Platform Project to Self-Hosted" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", + "title": "Migrate from Heroku to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", + "title": "Backup and Restore using the CLI" + } + ], + "resultChars": 78095 + }, + { + "source": "web_fetch", + "query": "What are the exact recommended commands and flags for restoring a pg_dump dump into a Supabase Postgres database? Include any notes about roles, ownership, privileges, extensions, schemas to exclude, and disabling triggers.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" + } + ], + "resultChars": 1424 + }, + { + "source": "web_fetch", + "query": "List any recent entries tagged breaking-change, especially anything related to the CLI, local development, `supabase start`, `supabase db` commands, database restores/migrations, or Postgres major versions.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1248 + } + ] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, "eval": "build-cli-001-bootstrap-app", "stage": "build", "product": [ @@ -105,35 +232,166 @@ "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "name": "supabase project initialised (supabase/config.toml exists)", + "passed": true + }, + { + "name": "todos table is created by a migration file", + "passed": true + }, + { + "name": "todos table exists with at least 2 seeded rows", + "passed": true, + "notes": "found 2 rows" + }, + { + "name": "row level security is enabled on todos", + "passed": true + }, + { + "name": "a SELECT policy targets the authenticated role", + "passed": true + }, + { + "name": "REST API returns no todos to anonymous requests", + "passed": true, + "notes": "0 rows" + }, + { + "name": "REST API returns the todos to authenticated requests", + "passed": true, + "notes": "2 rows" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", + "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "claude-code-opus-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user d720a18b-4270-462b-bc8d-62f6f9792f54, signUp returned {\"userId\":\"d720a18b-4270-462b-bc8d-62f6f9792f54\"}" }, { - "name": "todos table is created by a migration file", - "passed": true + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "wrong password is rejected gracefully (no throw, no session)", "passed": true, - "notes": "found 2 rows" + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"d720a18b-4270-462b-bc8d-62f6f9792f54\"}" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "app code does not use the secret / service-role key", "passed": true, - "notes": "error 42501: permission denied for table todos" + "notes": "no secret-key references found" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "implementation uses @supabase/supabase-js", "passed": true, - "notes": "2 rows" + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -148,33 +406,53 @@ "docs": { "calls": [ { - "source": "web_fetch", - "query": "List any entries tagged breaking-change, and anything relevant to: local development with the CLI, database migrations, RLS policies, the anon/authenticated roles, API keys (publishable/secret vs anon/service_role), seed files, or config.toml. Include dates and links.", - "hasContent": true, + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md 2>&1 | head -60", "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 8255 + "resultChars": 3161 }, { - "source": "web_fetch", - "query": "Show the full recommended workflow and exact SQL for exposing a table in the public schema to the Data API, including GRANT statements for anon and authenticated roles, enabling RLS, and read-only access patterns. Quote the SQL verbatim.", + "source": "shell_fetch", + "query": "curl -s https://supabase.com/changelog.md | grep -n -i -B2 'Breaking Change · .*\\(Auth\\|auth\\)' | head -60", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1353 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"signUp email password options data user metadata javascript\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/api/securing-your-api.md" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" } ], - "resultChars": 1245 + "resultChars": 22309 } ] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-opus-5/build-auth-001-email-password-flow.json" }, { "experiment": "claude-code-opus-5", @@ -436,53 +714,88 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"secret key service_role server-side createClient node backend persistSession\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" + } + ], + "resultChars": 45578 + } + ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-4.8/build-dataapi-001-relational-report.json" + "sourcePath": "claude-code-opus-5/build-dataapi-001-relational-report.json" }, { - "experiment": "claude-code-opus-4.8", + "experiment": "claude-code-opus-5", "experimentSuite": "benchmark", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-4-8", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -495,67 +808,12 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"restore pg_dump custom format dump into Supabase migrate existing postgres database\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", - "title": "Migrate from Vercel Postgres to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/neon", - "title": "Migrate from Neon to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/restore-from-platform", - "title": "Restore a Platform Project to Self-Hosted" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/heroku", - "title": "Migrate from Heroku to Supabase" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-within-supabase/backup-restore", - "title": "Backup and Restore using the CLI" - } - ], - "resultChars": 78095 - }, - { - "source": "web_fetch", - "query": "What are the exact recommended commands and flags for restoring a pg_dump dump into a Supabase Postgres database? Include any notes about roles, ownership, privileges, extensions, schemas to exclude, and disabling triggers.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres.md" - } - ], - "resultChars": 1424 - }, - { - "source": "web_fetch", - "query": "List any recent entries tagged breaking-change, especially anything related to the CLI, local development, `supabase start`, `supabase db` commands, database restores/migrations, or Postgres major versions.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1248 - } - ] + "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-opus-5/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-opus-5", @@ -2382,148 +2640,81 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [] - }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-opus-5/resolve-security-002-rls-cross-tenant-leak.json" - }, - { - "experiment": "claude-code-opus-5-no-skills", - "experimentSuite": "no-skills", - "experimentDisplay": { - "agent": "claude-code", - "modelProvider": "anthropic", - "modelId": "claude-opus-5", - "reasoningEffort": "high" - }, - "eval": "build-auth-001-email-password-flow", - "stage": "build", - "product": [ - "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": true, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user db370024-8f1e-4509-836e-53f273ac466c, signUp returned {\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" - } - ], - "skills": { - "available": [], - "loaded": [] + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [] }, - "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", - "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-4.8-no-skills/build-auth-001-email-password-flow.json" + "sourcePath": "claude-code-opus-5/resolve-security-002-rls-cross-tenant-leak.json" }, { - "experiment": "claude-code-opus-4.8-no-skills", + "experiment": "claude-code-opus-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-4-8", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-cli-001-bootstrap-app", + "eval": "build-auth-001-email-password-flow", "stage": "build", "product": [ - "database", - "data-api" + "auth", + "database" ], "topic": [ - "migrations", + "sdk", "rls" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "supabase project initialised (supabase/config.toml exists)", - "passed": true + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" }, { - "name": "todos table is created by a migration file", - "passed": true + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 6b4b33dc-03b1-440f-b7d8-0c2eaf088d50, signUp returned {\"userId\":\"6b4b33dc-03b1-440f-b7d8-0c2eaf088d50\"}" }, { - "name": "todos table exists with at least 2 seeded rows", + "name": "signup metadata reaches the profile (display name)", "passed": true, - "notes": "found 2 rows" + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "row level security is enabled on todos", - "passed": true + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "a SELECT policy targets the authenticated role", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"6b4b33dc-03b1-440f-b7d8-0c2eaf088d50\"}" }, { - "name": "REST API returns no todos to anonymous requests", + "name": "getMyProfile returns the signed-in user's profile", "passed": true, - "notes": "0 rows" + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "REST API returns the todos to authenticated requests", + "name": "app code does not use the secret / service-role key", "passed": true, - "notes": "2 rows" + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -2533,10 +2724,10 @@ "docs": { "calls": [] }, - "prompt": "We're kicking off a todos app and I want the Supabase side ready for the team\nto build on. Set it up the way we'd run it in development, with schema changes\ntracked as migrations so they can be reviewed and replayed.\n\nFor the first slice we just need a `todos` table. Todos aren't public: anyone\nsigned in can read all of them, but nothing should be writable through the API\nfor now. Add a couple of sample todos so there's something to look at.\n\nBefore you hand it back, make sure the running API actually behaves that way —\nsigned-in users get the todos, signed-out requests get nothing.", - "promptSourcePath": "evals/build-cli-001-bootstrap-app/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-cli-001-bootstrap-app.json" + "sourcePath": "claude-code-opus-5-no-skills/build-auth-001-email-password-flow.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -2755,48 +2946,55 @@ "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-001-relational-report.json" + "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-001-relational-report.json" }, { - "experiment": "claude-code-opus-4.8-no-skills", + "experiment": "claude-code-opus-5-no-skills", "experimentSuite": "no-skills", "experimentDisplay": { "agent": "claude-code", "modelProvider": "anthropic", - "modelId": "claude-opus-4-8", + "modelId": "claude-opus-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -2806,10 +3004,10 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-opus-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-opus-5-no-skills/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-opus-5-no-skills", @@ -3989,7 +4187,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user d85c49f2-9854-4de0-9395-9b4048713489, signUp returned {\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" + "notes": "db user 00a4b125-5d0f-4e3e-96a8-7df8041aa694, signUp returned {\"userId\":\"00a4b125-5d0f-4e3e-96a8-7df8041aa694\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -4004,7 +4202,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" + "notes": "{\"userId\":\"00a4b125-5d0f-4e3e-96a8-7df8041aa694\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -4348,7 +4546,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -4358,6 +4558,70 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -6163,7 +6427,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 175ffe86-cecb-4504-b9b2-843114d08b2c, signUp returned {\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" + "notes": "db user af7cdf33-fe1f-4c8a-9285-3691de4a9062, signUp returned {\"userId\":\"af7cdf33-fe1f-4c8a-9285-3691de4a9062\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6178,7 +6442,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" + "notes": "{\"userId\":\"af7cdf33-fe1f-4c8a-9285-3691de4a9062\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6440,6 +6704,67 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -7512,7 +7837,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 3251eadd-3f5a-452a-9085-d7e690fac0df, signUp returned {\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" + "notes": "db user eae71afb-4f4b-42c1-9749-cf6007ba7c10, signUp returned {\"userId\":\"eae71afb-4f4b-42c1-9749-cf6007ba7c10\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -7527,7 +7852,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" + "notes": "{\"userId\":\"eae71afb-4f4b-42c1-9749-cf6007ba7c10\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -7551,19 +7876,15 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog.md breaking-change auth supabase-js", - "pages": [] - }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession createClient auth local storage browser client\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", + "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession authenticated profile table select\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n totalCount\n }\n}", "hasContent": true, "pages": [ { @@ -7571,27 +7892,26 @@ "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", - "title": "Login with GitHub" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", - "title": "Login with X / Twitter" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" } ], - "resultChars": 74654 + "resultChars": 116820 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js auth.signUp signInWithPassword getUser select profile from table reference javascript\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", + "query": "query {\n searchDocs(query: \"site: supabase.com/docs/reference/javascript auth signUp signInWithPassword user_metadata display_name\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n totalCount\n }\n}", "hasContent": true, "pages": [ { @@ -7599,66 +7919,44 @@ "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { "url": "https://supabase.com/docs/guides/auth/passwords", "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" }, { - "url": "https://supabase.com/docs/reference/dart/auth-signup" + "url": "https://supabase.com/docs/reference/javascript/auth-signup" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - } - ], - "resultChars": 163499 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"reference javascript auth signInWithPassword\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" + "url": "https://supabase.com/docs/guides/auth/auth-identity-linking", + "title": "Identity Linking" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + "url": "https://supabase.com/docs/guides/auth/quickstarts/astrojs", + "title": "Use Supabase Auth with Astro" } ], - "resultChars": 3390 + "resultChars": 186803 } ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" }, { @@ -8016,7 +8314,8 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { @@ -8027,6 +8326,73 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -10331,214 +10697,64 @@ "stage": "build", "product": [ "auth", - "database" - ], - "topic": [ - "sdk", - "rls" - ], - "suite": "benchmark", - "interface": "cli", - "cliVersion": "2.109.1", - "passed": false, - "checks": [ - { - "name": "auth module loads and the driver completes", - "passed": true, - "notes": "driver produced a result" - }, - { - "name": "signUp creates the account and returns its user id", - "passed": true, - "notes": "db user 40f508e4-cd55-4379-9f5a-3c9392f94902, signUp returned {\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" - }, - { - "name": "signup metadata reaches the profile (display name)", - "passed": true, - "notes": "profiles.display_name = \"Alex Doe\"" - }, - { - "name": "wrong password is rejected gracefully (no throw, no session)", - "passed": true, - "notes": "{\"error\":\"Invalid login credentials\"}" - }, - { - "name": "signIn with the right password returns the user id", - "passed": true, - "notes": "{\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" - }, - { - "name": "getMyProfile returns the signed-in user's profile", - "passed": true, - "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" - }, - { - "name": "app code does not use the secret / service-role key", - "passed": true, - "notes": "no secret-key references found" - }, - { - "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" - } - ], - "skills": { - "available": [], - "loaded": [] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"auth sign up with metadata password grant profiles row level security local supabase auth\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - } - ], - "resultChars": 117348 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"auth signup options data signInWithPassword access token refresh token user endpoint REST\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions/implicit-flow", - "title": "Implicit flow" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - } - ], - "resultChars": 104157 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"JavaScript reference get user current session auth user endpoint\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getsession" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/egress", - "title": "Manage Egress usage" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-currentuser" - } - ], - "resultChars": 130065 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"refresh token grant_type refresh_token supabase auth v1 token json body\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login", - "title": "Social Login" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - } - ], - "resultChars": 86530 - } - ] + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 31f642a8-b8cb-4a51-9506-fd7e53e58b4f, signUp returned {\"userId\":\"31f642a8-b8cb-4a51-9506-fd7e53e58b4f\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"31f642a8-b8cb-4a51-9506-fd7e53e58b4f\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -10813,6 +11029,67 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -12095,7 +12372,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 58379f63-4af9-429b-9ea2-ba595cbb73a8, signUp returned {\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" + "notes": "db user ed685aae-6c81-43fd-b019-bcf3b9bfc3a5, signUp returned {\"userId\":\"ed685aae-6c81-43fd-b019-bcf3b9bfc3a5\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -12110,7 +12387,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" + "notes": "{\"userId\":\"ed685aae-6c81-43fd-b019-bcf3b9bfc3a5\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -12134,79 +12411,57 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript auth signUp options data user metadata signInWithPassword getUser select profile RLS\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data user metadata signInWithPassword getUser profile table RLS auth.uid trigger new user\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/reference/swift/auth-signup", + "title": "signUp()" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signup" } ], - "resultChars": 55094 + "resultChars": 24252 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createClient signUp signInWithPassword select single auth session browser persistSession current\", limit: 8) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- CHANGELOG AUTH HITS ---'; curl -fsSL https://supabase.com/changelog.md | rg -i -C 2 'breaking-change|auth|signup|sign.?in' | head -n 120 || true; printf '%s\\\\n' '--- WORKSPACE FILES ---'; rg --files -g '\"'!node_modules'\"' | sort; printf '%s\\\\n' '--- AUTH ---'; sed -n '1,240p' app/src/auth.mjs; printf '%s\\\\n' '--- PACKAGE ---'; cat app/package.json; printf '%s\\\\n' '--- SUPABASE CONFIG ---'; sed -n '1,260p' supabase/config.toml; printf '%s\\\\n' '--- MIGRATIONS ---'; find supabase -maxdepth 3 -type f -print -exec sh -c 'case \\\"\"'$1\" in *.sql) echo \"--- $1\"; sed -n \"1,260p\" \"$1\";; esac'\"' sh {} \\\\;; printf '%s\\\\n' '--- STATUS ---'; supabase status\"", "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", - "title": "Login with X / Twitter" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", - "title": "Login with Zoom" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-kakao", - "title": "Login with Kakao" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-initialize" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 93981 + "resultChars": 1467 } ] }, @@ -13082,73 +13337,158 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js select nested relationships foreign key alias service role server Node\", limit: 5) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + }, + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 47636 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '---CHANGELOG---'; curl -fsSL https://supabase.com/changelog.md | head -n 120; printf '%s\\\\n' '---AGENTS---'; find .. -name AGENTS.md -print; printf '%s\\\\n' '---REPORT---'; sed -n '1,260p' app/report.mjs; printf '%s\\\\n' '---PACKAGE---'; for f in package.json app/package.json; do if [ -f \\\"\"'$f\" ]; then echo \"### $f\"; cat \"$f\"; fi; done; printf '\"'%s\\\\n' '---SUPABASE FILES---'; find supabase -maxdepth 3 -type f \"'! -path '\"'*/.temp/*' -print | sort; printf '%s\\\\n' '---SQL---'; for f in \"'$(find supabase -maxdepth 3 -type f '\"\\\\( -name '*.sql' -o -name 'config.toml' \\\\) \"'! -path '\"'*/.temp/*' | sort); do echo \\\"### \"'$f\"; sed -n '\"'1,300p' \\\"\"'$f\"; done'", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6428 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested relationships aggregate count sum foreign tables\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- app/restock.mjs ---' && sed -n '1,240p' app/restock.mjs && printf '%s\\\\n' '--- app/package.json ---' && cat app/package.json && printf '%s\\\\n' '--- migration ---' && sed -n '1,320p' supabase/migrations/0000_inventory_schema.sql && printf '%s\\\\n' '--- config ---' && sed -n '1,220p' supabase/config.toml && printf '%s\\\\n' '--- status ---' && supabase status && printf '%s\\\\n' '--- changelog relevant lines ---' && curl -fsSL https://supabase.com/changelog.md | rg -i -m 20 'breaking-change|postgrest|supabase-js|javascript'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" - }, - { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" - }, - { - "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", - "title": "How to do automatic retries with `supabase-js`" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 42632 + "resultChars": 5661 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"secret key backend Data API apikey Authorization header Supabase REST\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"Supabase JavaScript select foreign tables nested relationships createClient secret key backend Node\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" } ], - "resultChars": 65167 + "resultChars": 24344 } ] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" }, { "experiment": "codex-gpt-5.6", @@ -14960,7 +15300,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user b72ca89c-acb9-48dc-9a16-80af3c5b09ca, signUp returned {\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" + "notes": "db user c8115185-6cf0-4a18-9a3f-b354212375ca, signUp returned {\"userId\":\"c8115185-6cf0-4a18-9a3f-b354212375ca\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -14975,7 +15315,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" + "notes": "{\"userId\":\"c8115185-6cf0-4a18-9a3f-b354212375ca\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -15001,7 +15341,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp user metadata signInWithPassword auth getUser select profiles RLS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js createClient signUp email password user metadata signInWithPassword getUser select single profiles browser client\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -15009,23 +15349,67 @@ "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" + } + ], + "resultChars": 130679 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript auth signUp options data metadata signInWithPassword select maybeSingle\", limit: 10) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithsso" + }, + { + "url": "https://supabase.com/docs/reference/dart/using-modifiers-maybesingle" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinanonymously" } ], - "resultChars": 53768 + "resultChars": 19708 } ] }, @@ -15861,31 +16245,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase REST API JavaScript fetch apikey Authorization service role secret key Range header pagination PostgREST\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js Node select nested relationships service role key local development\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/api/handling-errors-in-supabase-js", - "title": "Handling errors in `supabase-js`" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" } ], - "resultChars": 39367 + "resultChars": 67379 } ] }, @@ -15894,6 +16278,96 @@ "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js select foreign table relationships environment variables service role Node.js\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", + "title": "Self-Hosted Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/integrations/vercel-marketplace", + "title": "Vercel Marketplace" + } + ], + "resultChars": 70905 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", @@ -17383,68 +17857,149 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "medium" }, - "eval": "resolve-security-002-rls-cross-tenant-leak", - "stage": "resolve", + "eval": "resolve-security-002-rls-cross-tenant-leak", + "stage": "resolve", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "RLS enabled on notes", + "passed": true + }, + { + "name": "tenant A sees only org A notes", + "passed": true + }, + { + "name": "tenant B cannot read org A notes", + "passed": true + }, + { + "name": "tenant A author can update own note", + "passed": true + }, + { + "name": "tenant B cannot update org A note", + "passed": true + }, + { + "name": "tenant B author can delete own note", + "passed": true + }, + { + "name": "tenant B cannot delete org A note", + "passed": true + }, + { + "name": "tenant A can insert note in own org", + "passed": true + }, + { + "name": "tenant B cannot insert into org A", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", "product": [ - "database", - "auth" + "auth", + "database" ], "topic": [ - "rls", - "security" + "sdk", + "rls" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "RLS enabled on notes", - "passed": true - }, - { - "name": "tenant A sees only org A notes", - "passed": true + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" }, { - "name": "tenant B cannot read org A notes", - "passed": true + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 78735a72-891a-4831-b74d-a7eacd855314, signUp returned {\"userId\":\"78735a72-891a-4831-b74d-a7eacd855314\"}" }, { - "name": "tenant A author can update own note", - "passed": true + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" }, { - "name": "tenant B cannot update org A note", - "passed": true + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" }, { - "name": "tenant B author can delete own note", - "passed": true + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"78735a72-891a-4831-b74d-a7eacd855314\"}" }, { - "name": "tenant B cannot delete org A note", - "passed": true + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" }, { - "name": "tenant A can insert note in own org", - "passed": true + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" }, { - "name": "tenant B cannot insert into org A", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { - "available": [], - "loaded": [] + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.6-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "opencode-kimi-k3/build-auth-001-email-password-flow.json" }, { "experiment": "opencode-kimi-k3", @@ -17624,17 +18179,200 @@ { "name": "pg_cron job 'enqueue-tasks' scheduled to run every minute", "passed": true, - "notes": "schedule='* * * * *', active=true" + "notes": "schedule='* * * * *', active=true" + }, + { + "name": "cron command enqueues to the 'tasks' queue", + "passed": true, + "notes": "queue depth 0 -> 1" + }, + { + "name": "process-tasks function drains the queue", + "passed": true, + "notes": "function removed the seeded message (id 37) from the queue" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"pg_cron schedule job send message to pgmq queue every minute\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" + }, + { + "url": "https://supabase.com/docs/guides/queues/pgmq", + "title": "PGMQ Extension" + }, + { + "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", + "title": "Automatic embeddings" + }, + { + "url": "https://supabase.com/docs/guides/functions/schedule-functions", + "title": "Scheduling Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/cron", + "title": "Cron" + } + ], + "resultChars": 68390 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"Supabase Queues pgmq consume messages edge function read delete\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", + "title": "Consuming Supabase Queue Messages with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/queues", + "title": "Supabase Queues" + }, + { + "url": "https://supabase.com/docs/guides/queues/api", + "title": "API" + }, + { + "url": "https://supabase.com/docs/guides/queues/quickstart", + "title": "Quickstart" + } + ], + "resultChars": 18714 + } + ] + }, + "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", + "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3/build-dataapi-001-relational-report.json" + }, + { + "experiment": "opencode-kimi-k3", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "cron command enqueues to the 'tasks' queue", - "passed": true, - "notes": "queue depth 0 -> 1" + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "process-tasks function drains the queue", + "name": "report queries via the Data API, not raw SQL", "passed": true, - "notes": "function removed the seeded message (id 37) from the queue" + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -17647,65 +18385,12 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"pg_cron schedule job send message to pgmq queue every minute\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_net", - "title": "pg_net: Async Networking" - }, - { - "url": "https://supabase.com/docs/guides/queues/pgmq", - "title": "PGMQ Extension" - }, - { - "url": "https://supabase.com/docs/guides/ai/automatic-embeddings", - "title": "Automatic embeddings" - }, - { - "url": "https://supabase.com/docs/guides/functions/schedule-functions", - "title": "Scheduling Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/cron", - "title": "Cron" - } - ], - "resultChars": 68390 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"Supabase Queues pgmq consume messages edge function read delete\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/queues/consuming-messages-with-edge-functions", - "title": "Consuming Supabase Queue Messages with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/queues", - "title": "Supabase Queues" - }, - { - "url": "https://supabase.com/docs/guides/queues/api", - "title": "API" - }, - { - "url": "https://supabase.com/docs/guides/queues/quickstart", - "title": "Quickstart" - } - ], - "resultChars": 18714 - } - ] + "calls": [] }, - "prompt": "I want to set up a recurring background workflow on my local Supabase stack.\n\nCan you set up a cron job called `enqueue-tasks` to run every minute and push a task into a queue called `tasks`? Then add a `process-tasks` edge function that reads messages off the `tasks` queue and removes them, so a scheduled worker can keep the backlog drained.", - "promptSourcePath": "evals/build-cli-003-pg-cron-queue-workflow/PROMPT.md", - "attempts": 1, - "sourcePath": "opencode-kimi-k3/build-cli-003-pg-cron-queue-workflow.json" + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3/build-dataapi-002-restock-alert-report.json" }, { "experiment": "opencode-kimi-k3", @@ -19118,6 +19803,82 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 2629a92e-db2f-495e-b419-0010702fc708, signUp returned {\"userId\":\"2629a92e-db2f-495e-b419-0010702fc708\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"2629a92e-db2f-495e-b419-0010702fc708\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "opencode-kimi-k3-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", @@ -19315,6 +20076,126 @@ "attempts": 1, "sourcePath": "opencode-kimi-k3-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "opencode-kimi-k3-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "opencode", + "modelProvider": "moonshotai", + "modelId": "moonshotai/kimi-k3" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "opencode-kimi-k3-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "opencode-kimi-k3-no-skills", "experimentSuite": "no-skills", diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 32fc1535..b3c61b73 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -644,7 +644,7 @@ "suite": "regression", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "posts script still lists published posts", @@ -665,6 +665,16 @@ "name": "public script does not hold the secret key", "passed": true, "notes": "posts.mjs holds no secret-key reference" + }, + { + "name": "posts script actually depends on its publishable key", + "passed": false, + "notes": "still produced [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"] with a corrupted publishable key — looks hardcoded or key-agnostic" + }, + { + "name": "stats script actually depends on its secret key", + "passed": false, + "notes": "still produced {\"drafts\":0} with a corrupted secret key — looks hardcoded or key-agnostic" } ], "skills": { @@ -677,40 +687,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable and secret API keys migration from anon and service_role\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - } - ], - "resultChars": 107177 - } - ] + "calls": [] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" }, { @@ -1182,7 +1163,7 @@ "suite": "regression", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "posts script still lists published posts", @@ -1203,6 +1184,16 @@ "name": "public script does not hold the secret key", "passed": true, "notes": "posts.mjs holds no secret-key reference" + }, + { + "name": "posts script actually depends on its publishable key", + "passed": false, + "notes": "still produced [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"] with a corrupted publishable key — looks hardcoded or key-agnostic" + }, + { + "name": "stats script actually depends on its secret key", + "passed": false, + "notes": "still produced {\"drafts\":0} with a corrupted secret key — looks hardcoded or key-agnostic" } ], "skills": { @@ -1214,7 +1205,7 @@ }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" }, {