feat: add Playwright test suite

.github/workflows/ci-pr.yml

@@ -4,6 +4,9 @@ on:
   pull_request:
     types: [opened, synchronize, reopened]
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -59,6 +62,9 @@ jobs:
   ci-hack:
     name: Cargo Hack
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
     steps:
       - uses: actions/checkout@v4
       - name: Install Rust
@@ -148,3 +154,170 @@ jobs:
           helm dependency update helm-charts/
           helm lint helm-charts/
           helm template test helm-charts/ > /dev/null
+
+  e2e-playwright:
+    name: E2E (Playwright)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+
+      - name: Install native deps for rdkafka
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            pkg-config \
+            libcurl4-openssl-dev \
+            libsasl2-dev \
+            zlib1g-dev \
+            libpq-dev
+
+      - uses: Swatinem/rust-cache@v2.7.7
+
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: just
+          checksum: true
+
+      - name: Install diesel CLI
+        run: cargo install diesel_cli --no-default-features --features postgres --locked
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Install e2e dependencies
+        run: npm ci
+
+      - name: Type check e2e suite
+        run: npm run typecheck
+
+      - name: Install dashboard dependencies
+        run: npm ci
+        working-directory: website
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium
+
+      - name: Build decision-engine
+        run: cargo build --no-default-features --features postgres
+
+      # Datastores only. oneclick.sh is a developer script — it kills processes on ports, prompts for
+      # confirmation, boots a docs preview and reinstalls dashboard deps — none of which CI wants.
+      # `--wait` blocks on the healthchecks already defined in docker-compose.yaml, so no custom
+      # polling is needed. kafka-init is excluded from --wait: it is a one-shot that creates topics
+      # and exits, which --wait would treat as a failure.
+      - name: Start datastores
+        run: |
+          COMPOSE_PROFILES= docker compose --profile postgres-ghcr --profile analytics-clickhouse \
+            up -d postgresql redis kafka kafka-init clickhouse mailpit
+          COMPOSE_PROFILES= docker compose --profile postgres-ghcr --profile analytics-clickhouse \
+            up -d --wait postgresql redis kafka clickhouse mailpit
+
+      # KNOWN BUG WORKAROUND: migrations_pg/00000000000000_diesel_postgresql_initial_setup creates 33
+      # tables (including merchant_account), but diesel reserves version 00000000000000 for its own
+      # setup migration and records it as applied when initialising a fresh database — so the repo's
+      # migration with that version is silently SKIPPED. Applying it explicitly first is the only way
+      # a clean Postgres gets the core schema. Remove this once the migration is renamed off 0000...
+      # psql comes from the postgres container, so the runner needs no postgresql-client package.
+      - name: Apply database schema
+        run: |
+          COMPOSE_PROFILES= docker compose --profile postgres-ghcr exec -T postgresql \
+            psql -v ON_ERROR_STOP=1 -U db_user -d decision_engine_db \
+            < migrations_pg/00000000000000_diesel_postgresql_initial_setup/up.sql
+          just migrate-pg
+
+      - name: Seed global service configs
+        run: |
+          COMPOSE_PROFILES= docker compose --profile postgres-ghcr exec -T postgresql \
+            psql -v ON_ERROR_STOP=1 -U db_user -d decision_engine_db -c "
+              INSERT INTO service_configuration (name, value)
+              SELECT v.name, v.value
+              FROM (VALUES
+                  ('ENABLE_MERCHANT_ON_VOLUME_DISTRIBUTION_FEATURE_SR_V3', '{\"enableAll\":true,\"enableAllRollout\":100}'),
+                  ('merchants_enabled_for_score_keys_unification',         '{\"enableAll\":true,\"enableAllRollout\":100}'),
+                  ('SR_V3_INPUT_CONFIG_DEFAULT',                           '{\"defaultLatencyThreshold\":90,\"defaultBucketSize\":125,\"defaultHedgingPercent\":5}')
+              ) AS v(name, value)
+              WHERE NOT EXISTS (
+                  SELECT 1 FROM service_configuration sc WHERE sc.name = v.name
+              );"
+
+      # Playwright starts the API and dashboard itself (see the webServer block in
+      # playwright.config.ts) and tears them down afterwards.
+      # Preflight: prove the API can actually serve a merchant creation before spending an hour on 220
+      # tests. Every failing test so far has died on exactly this call, so if it fails here we get the
+      # server's real response (or the hang) in seconds instead of inferring it from timeouts. Also
+      # dumps Postgres connection state, since the recurring Storage("db connection") warnings from the
+      # background workers point at the pool.
+      - name: Preflight — capture a backtrace of the hang
+        run: |
+          set -x
+          sudo apt-get update -qq && sudo apt-get install -y -qq gdb postgresql-client
+
+          ./target/debug/open_router &
+          SRV=$!
+          for i in $(seq 1 90); do curl -sf -o /dev/null http://localhost:8080/health && break; sleep 2; done
+          curl -sS -o /dev/null -w 'health=%{http_code} in %{time_total}s\n' http://localhost:8080/health
+
+          # Environment facts first — cheap, and each rules a class of cause in or out.
+          echo "--- resolution + listeners ---"
+          getent hosts localhost || true
+          ss -ltnp 2>/dev/null | grep -E ':5432|:8080' || true
+          echo "--- can the HOST reach postgres with the app's credentials? ---"
+          PGPASSWORD=db_pass pg_isready -h 127.0.0.1 -p 5432 -U db_user -d decision_engine_db || true
+          PGPASSWORD=db_pass psql -h 127.0.0.1 -U db_user -d decision_engine_db -tAqX \
+            -c "select 'host->pg ok', now();" || true
+
+          # Fire the hanging request in the background, then photograph the process while it is stuck.
+          echo "--- firing create (backgrounded) ---"
+          curl -sS -o /tmp/create.out -w '\ncreate=%{http_code} in %{time_total}s\n' --max-time 90 \
+            -X POST http://localhost:8080/merchant-account/create \
+            -H 'content-type: application/json' -H 'x-tenant-id: public' -H 'x-admin-secret: test_admin' \
+            -d '{"merchant_id":"preflight_probe","gateway_success_rate_based_decider_input":null}' &
+          CURL=$!
+          sleep 12   # sample the process mid-hang, well before bb8 gives up at 60s
+
+          echo "--- THREADS ---"
+          cat /proc/$SRV/status | grep -E '^(Threads|State)' || true
+          for t in /proc/$SRV/task/*; do echo "$(basename $t) $(cat $t/comm 2>/dev/null) $(cat $t/stat 2>/dev/null | awk '{print $3}')"; done || true
+
+          echo "--- BACKTRACE (all threads) ---"
+          # Debug build, so symbols are present and this should name the parked future directly.
+          # NOTE: do NOT pipe through tail — the frames that name the blocked work are at the TOP of
+          # each thread's stack (#0..#40); tailing keeps only the tokio scaffolding underneath them.
+          sudo gdb -p $SRV -batch -ex "set pagination off" -ex "thread apply all bt" 2>&1 || true
+
+          echo "--- postgres side ---"
+          PGPASSWORD=db_pass psql -h 127.0.0.1 -U db_user -d decision_engine_db \
+            -c "SELECT pid, state, wait_event_type, wait_event, left(query,60) AS q FROM pg_stat_activity;" || true
+
+          wait $CURL || true
+          cat /tmp/create.out || true
+          kill $SRV 2>/dev/null || true
+        env:
+          DECISION_ENGINE__LOG__CONSOLE__LEVEL: DEBUG
+
+      - name: Run Playwright e2e
+        run: npx playwright test
+        env:
+          CI: 'true'
+          DECISION_ENGINE__LOG__CONSOLE__LEVEL: WARN
+          PW_WORKERS: '1'
+
+      - name: Upload Playwright report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: |
+            playwright-report/
+            test-results/
+          retention-days: 7
+          if-no-files-found: ignore

.gitignore

@@ -10,6 +10,12 @@ dump.rdb
 cypress/screenshots
 cypress/videos
 cypress/fixtures
+
+# Playwright generated artifacts (HTML report, traces, screenshots)
+playwright-report/
+test-results/
+blob-report/
+.playwright/
 .mintlify-dev.log
 **/gsm.csv
 **/scratch

cypress/scripts/run-e2e.js

@@ -7,7 +7,7 @@ const path = require('path')
 const ROOT = path.resolve(__dirname, '../..')
 const VALID_MODES = new Set(['source', 'docker', 'all'])
 const DEFAULT_SPEC = 'cypress/e2e/**/*.cy.js'
-const READINESS_TIMEOUT_MS = 180000
+const READINESS_TIMEOUT_MS = Number(process.env.E2E_READINESS_TIMEOUT_MS) || 180000
 const READINESS_INTERVAL_MS = 2000
 const KNOWN_COMPOSE_PROJECTS = [
   'decision-engine',
@@ -24,16 +24,25 @@ const EXPECTED_CLICKHOUSE_TABLES = [
   'analytics_payment_audit_lookup_summaries',
 ]
 
+const VALID_RUNNERS = new Set(['playwright', 'cypress'])
+
 function parseArgs() {
   const args = process.argv.slice(2)
   const mode = args[0] || 'all'
   const keepAlive = args.includes('--keep-alive')
+  const runnerArg = args.find((a) => a.startsWith('--runner='))
+  // Playwright is the default runner; Cypress (frozen) is still selectable via --runner=cypress
+  // or E2E_RUNNER=cypress for the legacy suite.
+  const runner = (runnerArg ? runnerArg.split('=')[1] : process.env.E2E_RUNNER) || 'playwright'
 
   if (!VALID_MODES.has(mode)) {
     throw new Error(`Unsupported E2E mode '${mode}'. Use source, docker, or all.`)
   }
+  if (!VALID_RUNNERS.has(runner)) {
+    throw new Error(`Unsupported E2E runner '${runner}'. Use playwright or cypress.`)
+  }
 
-  return { mode, keepAlive }
+  return { mode, keepAlive, runner }
 }
 
 function sleep(ms) {
@@ -250,6 +259,43 @@ async function runCypress(runtime) {
   )
 }
 
+async function runPlaywright(runtime) {
+  console.log(`\n[${runtime.mode}] Running Playwright suite...`)
+
+  // Pass-through for extra CLI flags (e.g. sharding in CI): PLAYWRIGHT_ARGS="--shard=1/4".
+  const extraArgs = process.env.PLAYWRIGHT_ARGS
+    ? process.env.PLAYWRIGHT_ARGS.split(' ').filter(Boolean)
+    : []
+
+  await runCommand(
+    `playwright-${runtime.mode}`,
+    'npx',
+    ['playwright', 'test', ...extraArgs],
+    {
+      env: {
+        // playwright.config.ts reads these plain env vars (no CYPRESS_ prefix).
+        // This script already booted the API and dashboard, so tell Playwright not to start its own.
+        PW_NO_WEBSERVER: '1',
+        RUNTIME_MODE: runtime.mode,
+        API_BASE_URL: runtime.apiBaseUrl,
+        UI_BASE_URL: runtime.uiBaseUrl,
+        DOCS_BASE_URL: runtime.docsBaseUrl,
+        CLICKHOUSE_HTTP_URL: runtime.clickhouseHttpUrl,
+        CLICKHOUSE_DATABASE: runtime.clickhouseDatabase,
+        CLICKHOUSE_USER: runtime.clickhouseUser,
+        CLICKHOUSE_PASSWORD: runtime.clickhousePassword,
+      },
+    },
+  )
+}
+
+async function runTests(runtime, runner) {
+  if (runner === 'cypress') {
+    return runCypress(runtime)
+  }
+  return runPlaywright(runtime)
+}
+
 async function stopDockerServices(profile) {
   await runCommand(
     `docker-down-${profile}`,
@@ -301,7 +347,7 @@ async function killProcessGroup(child) {
   }
 }
 
-async function runSourceMode(keepAlive) {
+async function runSourceMode(keepAlive, runner) {
   const runtime = {
     mode: 'source',
     apiBaseUrl: 'http://localhost:8080',
@@ -330,13 +376,13 @@ async function runSourceMode(keepAlive) {
 
   try {
     await waitForRuntime(runtime, sourceProcess)
-    await runCypress(runtime)
+    await runTests(runtime, runner)
   } finally {
     await cleanup()
   }
 }
 
-async function runDockerMode(keepAlive) {
+async function runDockerMode(keepAlive, runner) {
   const runtime = {
     mode: 'docker',
     apiBaseUrl: 'http://localhost:8080',
@@ -365,21 +411,23 @@ async function runDockerMode(keepAlive) {
 
   try {
     await waitForRuntime(runtime)
-    await runCypress(runtime)
+    await runTests(runtime, runner)
   } finally {
     await cleanup()
   }
 }
 
 async function main() {
-  const { mode, keepAlive } = parseArgs()
+  const { mode, keepAlive, runner } = parseArgs()
   const modes = mode === 'all' ? ['source', 'docker'] : [mode]
 
+  console.log(`[E2E] runner=${runner} mode=${mode}`)
+
   for (const selectedMode of modes) {
     if (selectedMode === 'source') {
-      await runSourceMode(keepAlive)
+      await runSourceMode(keepAlive, runner)
     } else {
-      await runDockerMode(keepAlive)
+      await runDockerMode(keepAlive, runner)
     }
   }
 }

oneclick.sh

@@ -65,7 +65,6 @@ EXPECTED_CLICKHOUSE_TABLES=(
     cost_fee_model
     cost_fee_model_segment
     cost_bin_product
-    connector_markup_overlay
 )
 
 check_and_kill_ports() {
@@ -560,8 +559,8 @@ run_infra_checklist
 if ! check_clickhouse_schema; then
     echo ""
     echo "ClickHouse schema is incomplete — attempting to (re)create cost-ingestion tables..."
-    # The cost tables (cost_daily_stats / cost_fee_model / connector_markup_overlay from
-    # 035_cost_model.sh, cost_bin_product from 036, the piecewise cost_fee_model_segment from 037,
+    # The cost tables (cost_daily_stats / cost_fee_model from 035_cost_model.sh,
+    # cost_bin_product from 036, the piecewise cost_fee_model_segment from 037,
     # and the card_product ALTER migration in 038) are only auto-run by the container on a fresh
     # clickhouse-data volume. Every one is idempotent and non-destructive — the CREATEs are
     # IF NOT EXISTS, and 038 is ADD COLUMN IF NOT EXISTS + a same-key MODIFY ORDER BY (a metadata-only