Systemsaholic · Systemsaholic · Jul 5, 2026 · Jul 5, 2026
diff --git a/.github/workflows/prod-health-monitor.yml b/.github/workflows/prod-health-monitor.yml
@@ -9,12 +9,14 @@ name: Prod Health Monitor
 #                     gracefully if absent).
 #   3. DB headroom  — prod active connections vs max_connections (requires the
 #                     PROD_DATABASE_URL repo secret, read-only SELECT; skipped if absent).
-#   4. Auth failures — a spike of failed-auth (401/403) events in security_audit_logs
-#                     within the lookback window: alerts when a SINGLE source IP crosses
+#   4. Auth failures — a spike of failed-auth events in security_audit_logs within the
+#                     lookback window: alerts when a SINGLE source IP crosses
 #                     AUTHFAIL_IP_THRESHOLD (targeted brute-force) or the total crosses
-#                     AUTHFAIL_TOTAL_THRESHOLD (distributed probing). Read-only SELECT on
-#                     the same PROD_DATABASE_URL secret; skipped if absent. (See #1419 —
-#                     this surfaces the 401/403 stream we already capture but never watched.)
+#                     AUTHFAIL_TOTAL_THRESHOLD (distributed probing). The total now
+#                     combines API 401/403 events (Phase 1, IP-bearing) with hook-sourced
+#                     GoTrue wrong-password / failed-MFA events (Phase 2, #1419 — no ip,
+#                     so total-only, never in the per-IP CTE). Read-only SELECT on the same
+#                     PROD_DATABASE_URL secret; skipped if absent.
 # On ANY problem it opens (or refreshes) ONE durable GitHub issue so there is an
 # off-hours alert; it auto-resolves (comment + close) when everything passes again.
 # Dedupe + no-spam logic lives in .github/scripts/health-alert.cjs (unit-tested).
@@ -49,7 +51,7 @@ env:
   # above normal (targeted brute-force ≈ tens of tries; distributed probing ≈ hundreds).
   AUTHFAIL_LOOKBACK_MIN: '20'      # slightly > cron interval to avoid gaps
   AUTHFAIL_IP_THRESHOLD: '20'      # failures from ONE source IP in the window → brute-force
-  AUTHFAIL_TOTAL_THRESHOLD: '150'  # total 401/403 in the window → distributed probing
+  AUTHFAIL_TOTAL_THRESHOLD: '150'  # combined total (API 401/403 + GoTrue pw/mfa fails) → distributed probing
 
 jobs:
   monitor:
@@ -186,10 +188,15 @@ jobs:
             { echo "detail<<EOF"; echo "skipped (no PROD_DATABASE_URL secret)"; echo "EOF"; } >> "$GITHUB_OUTPUT"
             echo "Auth: skipped (no secret)"; exit 0
           fi
-          # One read-only query returns: total failed-auth events in the window, and the
-          # single busiest source IP + its count. Bounded (connect/statement timeout +
-          # `timeout 30s`) exactly like the DB headroom check so a DB hang degrades to
-          # "report, don't false-page" rather than blocking the reconcile step.
+          # One read-only query returns: (1) total API failed-auth (401/403) events in
+          # the window, (2) the single busiest source IP + its count, and (3) a scalar
+          # count of hook-sourced GoTrue verify-fails (security.password_failed +
+          # security.mfa_failed, #1419 Phase 2). The hook events carry NO ip (the
+          # Supabase verification-hook payload is user-scoped), so they feed ONLY the
+          # total-threshold count and are deliberately kept OUT of the per-IP CTE.
+          # Bounded (connect/statement timeout + `timeout 30s`) exactly like the DB
+          # headroom check so a DB hang degrades to "report, don't false-page" rather
+          # than blocking the reconcile step.
           row="$(PGCONNECT_TIMEOUT=10 PGOPTIONS='-c statement_timeout=10000' \
             timeout 30s psql "$PROD_DATABASE_URL" -tA -F'|' -c \
             "WITH w AS (
@@ -200,24 +207,37 @@ jobs:
              ), top AS (
                SELECT ip, count(*) AS c FROM w
                WHERE ip IS NOT NULL AND ip <> '' GROUP BY ip ORDER BY c DESC LIMIT 1
+             ), h AS (
+               SELECT count(*) AS c
+               FROM public.security_audit_logs
+               WHERE event IN ('security.password_failed','security.mfa_failed')
+                 AND created_at > now() - make_interval(mins => ${AUTHFAIL_LOOKBACK_MIN})
              )
              SELECT (SELECT count(*) FROM w),
                     COALESCE((SELECT ip FROM top), '(none)'),
-                    COALESCE((SELECT c FROM top), 0);" \
+                    COALESCE((SELECT c FROM top), 0),
+                    (SELECT c FROM h);" \
             2>/dev/null || echo '')"
           total="$(echo "$row" | awk -F'|' '{print $1}')"
           top_ip="$(echo "$row" | awk -F'|' '{print $2}')"
           top_ct="$(echo "$row" | awk -F'|' '{print $3}')"
+          hook_ct="$(echo "$row" | awk -F'|' '{print $4}')"
           # Numeric-safe: a failed/empty query degrades (report, not alert).
           case "${total}" in ''|*[!0-9]*) total='' ;; esac
           case "${top_ct}" in ''|*[!0-9]*) top_ct='' ;; esac
+          # A missing/malformed hook count must never break the existing check —
+          # degrade it to 0 (do not inflate, do not fail the whole step).
+          case "${hook_ct}" in ''|*[!0-9]*) hook_ct='0' ;; esac
           if [ -z "$total" ] || [ -z "$top_ct" ]; then
             echo "ok=true" >> "$GITHUB_OUTPUT"
             { echo "detail<<EOF"; echo "auth-failure query failed/empty (check degraded, not alerting)"; echo "EOF"; } >> "$GITHUB_OUTPUT"
             echo "Auth: query failed — not alerting"; exit 0
           fi
-          summary="${total} failed-auth (401/403) in last ${AUTHFAIL_LOOKBACK_MIN}m; busiest IP ${top_ip}=${top_ct}"
-          if [ "$top_ct" -ge "$AUTHFAIL_IP_THRESHOLD" ] || [ "$total" -ge "$AUTHFAIL_TOTAL_THRESHOLD" ]; then
+          # Grand total feeding the distributed-probing threshold = API 401/403 plus
+          # hook-sourced GoTrue verify-fails (wrong-password / failed-MFA).
+          grand_total="$(( total + hook_ct ))"
+          summary="${grand_total} failed-auth in last ${AUTHFAIL_LOOKBACK_MIN}m (${total} API 401/403 + ${hook_ct} GoTrue pw/mfa); busiest IP ${top_ip}=${top_ct}"
+          if [ "$top_ct" -ge "$AUTHFAIL_IP_THRESHOLD" ] || [ "$grand_total" -ge "$AUTHFAIL_TOTAL_THRESHOLD" ]; then
             echo "ok=false" >> "$GITHUB_OUTPUT"
             { echo "detail<<EOF"; echo "auth-failure spike — ${summary} (IP>=${AUTHFAIL_IP_THRESHOLD} or total>=${AUTHFAIL_TOTAL_THRESHOLD})"; echo "EOF"; } >> "$GITHUB_OUTPUT"
           else

diff --git a/apps/api/src/security-audit/security-audit.types.ts b/apps/api/src/security-audit/security-audit.types.ts
@@ -5,3 +5,37 @@ export interface SecurityAuditEvent {
   agencyId?: string | null
   metadata?: Record<string, unknown>
 }
+
+/**
+ * Known `security.*` event names written to `public.security_audit_logs`.
+ *
+ * There are two distinct producers:
+ *
+ * 1. API-sourced (this Nest service) — emitted by `SecurityExceptionFilter`
+ *    for 401/403 on our API. These carry `metadata.ip`.
+ *      - `security.login_failed`   (401)
+ *      - `security.access_denied`  (403)
+ *
+ * 2. HOOK-SOURCED (#1419 Phase 2) — written directly by the Supabase Auth Hook
+ *    Postgres functions `public.hook_password_verification_attempt` /
+ *    `public.hook_mfa_verification_attempt` when GoTrue reports a failed
+ *    verification. These NEVER pass through this service, carry NO `ip`
+ *    (the hook payload is user-scoped only), and their `metadata` is the raw
+ *    GoTrue event payload.
+ *      - `security.password_failed`
+ *      - `security.mfa_failed`
+ *
+ * `SecurityAuditEvent.event` is a plain string so producers are not coupled to
+ * this list; it exists so consumers share one canonical enumeration.
+ */
+export const KNOWN_SECURITY_AUDIT_EVENTS = {
+  loginFailed: 'security.login_failed',
+  accessDenied: 'security.access_denied',
+  /** Hook-sourced: written by the Supabase password-verification hook, not this service. */
+  passwordFailed: 'security.password_failed',
+  /** Hook-sourced: written by the Supabase MFA-verification hook, not this service. */
+  mfaFailed: 'security.mfa_failed',
+} as const
+
+export type KnownSecurityAuditEvent =
+  (typeof KNOWN_SECURITY_AUDIT_EVENTS)[keyof typeof KNOWN_SECURITY_AUDIT_EVENTS]
diff --git a/docs/MONITORING.md b/docs/MONITORING.md
@@ -161,6 +161,46 @@ The scheduled Production health monitor is the off-hours backstop. During workin
 
 ---
 
+## Failed-Auth & Brute-Force Monitoring (#1419)
+
+Failed-authentication is watched by **Check 4 (Auth failures)** in the
+`Prod Health Monitor` workflow (`.github/workflows/prod-health-monitor.yml`). It runs a
+bounded, read-only SELECT against `public.security_audit_logs` every run and pages via
+the single durable alert issue when a spike is detected:
+
+- **Per-IP threshold** (`AUTHFAIL_IP_THRESHOLD`, default 20) — one source IP crossing it
+  in the window looks like targeted brute-force.
+- **Total threshold** (`AUTHFAIL_TOTAL_THRESHOLD`, default 150) — the combined failure
+  count crossing it looks like distributed probing.
+
+### What is captured
+
+| Event | Producer | Has `ip`? | Feeds |
+|-------|----------|-----------|-------|
+| `security.login_failed` | API `SecurityExceptionFilter` (401) | yes | per-IP + total |
+| `security.access_denied` | API `SecurityExceptionFilter` (403) | yes | per-IP + total |
+| `security.password_failed` | Supabase password-verification hook | no | total only |
+| `security.mfa_failed` | Supabase MFA-verification hook | no | total only |
+
+**Phase 1** (PR #1420) shipped the API-sourced 401/403 stream. **Phase 2** (#1419) closes
+the gap where wrong-password and failed-MFA attempts are handled entirely inside Supabase
+GoTrue — they never reach our API and `auth.audit_log_entries` is empty in prod. Two
+non-blocking, fail-open Postgres Auth Hook functions
+(`public.hook_password_verification_attempt` / `public.hook_mfa_verification_attempt`,
+migration `20260705130000_1419_auth_verification_hooks.sql`) now record a
+`security_audit_logs` row on failure at the GoTrue source. Because the hook payload is
+user-scoped (no IP), these events feed the **total** count only and are deliberately kept
+out of the per-IP CTE.
+
+> **Enablement is not automatic.** Shipping the migration creates the functions but does
+> nothing until the hooks are enabled in the Supabase Dashboard (Authentication > Hooks).
+> This is **staging-first and Al-gated** — see
+> [Enable Supabase Auth Verification Hooks](./runbooks/enable-supabase-auth-verification-hooks.md).
+> The hooks are monitoring-only and always return `{"decision":"continue"}`; they can
+> never block a login.
+
+---
+
 ## Known DB Constraints to Watch
 
 These constraints have caused errors in testing. When Sentry reports a `500` error from a relevant endpoint, check whether the constraint listed below is the cause.

diff --git a/docs/runbooks/enable-supabase-auth-verification-hooks.md b/docs/runbooks/enable-supabase-auth-verification-hooks.md
@@ -0,0 +1,124 @@
+# Enable Supabase Auth Verification Hooks (#1419 Phase 2)
+
+Turns on the two **Password / MFA Verification Attempt** Auth Hooks so failed human
+logins and failed MFA challenges — which are handled entirely inside Supabase GoTrue and
+never reach our API — get recorded in `public.security_audit_logs` and picked up by the
+`Prod Health Monitor` brute-force check (Check 4).
+
+**Staging-first and Al-gated.** Do the whole flow on **Staging** first, confirm both a
+deliberate failure lands a row AND normal login still works, then repeat on **Prod** only
+with Al's go-ahead.
+
+---
+
+## What is already shipped in code
+
+Migration `packages/database/src/migrations/20260705130000_1419_auth_verification_hooks.sql`
+(deployed via CI on merge to `main`) creates two **non-blocking, fail-open** Postgres
+functions and their grants:
+
+- `public.hook_password_verification_attempt(event jsonb)` — payload `{"user_id","valid"}`
+- `public.hook_mfa_verification_attempt(event jsonb)` — payload `{"factor_id","user_id","valid"}`
+
+Both:
+
+- INSERT a `security_audit_logs` row **only on failure** (`valid` is not `true`):
+  `security.password_failed` / `security.mfa_failed`, with the raw GoTrue payload as
+  `metadata`.
+- Wrap the INSERT in an exception-swallowing block so a logging error can **never** block
+  auth (fail-open).
+- **ALWAYS** return `jsonb_build_object('decision','continue')` — they are monitoring-only
+  and never reject, rate-limit, or otherwise interfere with a login.
+
+The migration creating the functions does **nothing** until the hooks are enabled in the
+Dashboard (below).
+
+> ⚠️ **Plan check:** confirm Auth Hooks (Password / MFA Verification Attempt) are available
+> on our Supabase plan tier before starting — some hooks require **Pro**. Coordinator / Al
+> to confirm per project. If unavailable, stop here and flag it; the migration is inert and
+> harmless in the meantime.
+
+---
+
+## Enablement steps (do on Staging first)
+
+Target project by environment:
+
+| Environment | Supabase project | Ref |
+|-------------|------------------|-----|
+| Staging | Tailfire-Staging | `sivpkddntsxbbcryjvnn` |
+| Prod | Tailfire-Prod | `cmktvanwglszgadjrorm` |
+
+### 1. Confirm the migration applied
+
+The functions ship with the branch's normal deploy (`deploy-staging.yml` → Staging,
+`deploy-prod.yml` → Prod). Verify they exist (read-only):
+
+```sql
+SELECT proname
+FROM pg_proc
+WHERE proname IN ('hook_password_verification_attempt','hook_mfa_verification_attempt');
+-- expect both rows
+```
+
+### 2. Enable the hooks in the Dashboard
+
+**Where:** Supabase Dashboard → the target project → **Authentication → Hooks**.
+
+- Set **Password Verification Attempt** → Postgres function
+  `public.hook_password_verification_attempt`.
+- Set **MFA Verification Attempt** → Postgres function
+  `public.hook_mfa_verification_attempt`.
+
+Save each.
+
+### 3. Verify a deliberate failure lands a row AND login still works
+
+- Attempt a login with a **wrong password** for a known test user.
+  - The login should be **rejected as normal** (the hook does not change the outcome).
+  - A `security.password_failed` row should appear:
+    ```sql
+    SELECT event, user_id, metadata, created_at
+    FROM public.security_audit_logs
+    WHERE event = 'security.password_failed'
+    ORDER BY created_at DESC
+    LIMIT 5;
+    ```
+- If the account has MFA, attempt a **wrong MFA code** and confirm a `security.mfa_failed`
+  row appears the same way.
+- Then log in with the **correct** password (and correct MFA) and confirm **login succeeds
+  normally** — this is the load-bearing check that the hook is non-blocking. No row is
+  written on success.
+
+### 4. Confirm the grant survived
+
+The hook runs as `supabase_auth_admin`; it needs INSERT on the table. Verify:
+
+```sql
+SELECT has_table_privilege('supabase_auth_admin', 'public.security_audit_logs', 'INSERT') AS can_insert;
+-- expect: t
+```
+
+(If `can_insert` is `f`, re-apply the migration — the grant block is idempotent.)
+
+### 5. Repeat on Prod (Al-gated)
+
+Only after Staging passes all of the above, and with Al's explicit go-ahead, repeat
+steps 1–4 against **Tailfire-Prod** (`cmktvanwglszgadjrorm`).
+
+---
+
+## Rollback
+
+To disable, set both hooks back to **None** in Authentication → Hooks. GoTrue stops calling
+the functions immediately; no login behavior changes (the hooks were non-blocking anyway).
+The functions and grants can be left in place — they are inert when not wired.
+
+---
+
+## Related
+
+- [Monitoring Guide → Failed-Auth & Brute-Force Monitoring](../MONITORING.md#failed-auth--brute-force-monitoring-1419)
+- `.github/workflows/prod-health-monitor.yml` (Check 4)
+- Migration `packages/database/src/migrations/20260705130000_1419_auth_verification_hooks.sql`
+- [B6 — Supabase Auth Production Configuration](./b6-supabase-auth-prod-config.md)