feat(ai-review): model cascade + delta/OSH/Test Plan CI

.github/workflows/selftest.yml

@@ -19,7 +19,9 @@ concurrency:
 jobs:
   review:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    # Fan-out (Opus + Sonnet Tasks) regularly exceeds 25m wall-clock; 30m was
+    # cancelling mid-review before Publish. Budget covers review + repair/retry.
+    timeout-minutes: 55
     permissions:
       contents: read
       pull-requests: write
@@ -57,16 +59,62 @@ jobs:
         uses: ./ai-review
         with:
           # Deliberately does NOT pass enable-context-stage: this self-test must
-          # exercise the same default configuration consumers get, and the
-          # context stage is on by default. A temporary "false" here during the
-          # 2026-08-09 stall diagnosis meant the default path went untested
-          # while it was live for every consumer — don't reintroduce that.
+          # exercise the same default configuration consumers get. Haiku Context
+          # still auto-skips on delta + K≤1 (deterministic context.md remains).
+          # Do not force 'false' here — that would leave the default path
+          # untested for full/K>1 reviews.
           pr-number: ${{ inputs.pr_number }}
           app-id: ${{ vars.MTM_BOT_APP_ID }}
           private-key: ${{ secrets.MTM_BOT_APP_PRIVATE_KEY }}
           anthropic-auth-token: ${{ secrets.ANTHROPIC_AUTH_TOKEN }}
           anthropic-base-url: ${{ vars.ANTHROPIC_BASE_URL }}
 
+      # Pins the delta baseline contract without a second push (which would
+      # re-enter this workflow and race cancel-in-progress). Unit tests cover
+      # prior_head…HEAD range selection with a real git history; this step
+      # asserts Publish actually stamped parseable meta on the live review.
+      - name: Assert published ai-review-meta
+        if: steps.r.outputs.verdict != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number }}
+          REPO: ${{ github.repository }}
+          EXPECT_HEAD: ${{ github.event.pull_request.head.sha }}
+          # Bound via env (never ${{ }} inside run:) so zizmor template-injection
+          # stays clean — verdict is only pass|fail from Publish, but the audit
+          # treats any in-run expansion as attacker-controllable.
+          VERDICT: ${{ steps.r.outputs.verdict }}
+        run: |
+          set -euo pipefail
+          if [ -z "${EXPECT_HEAD}" ]; then
+            EXPECT_HEAD="$(gh pr view "${PR_NUMBER}" --repo "${REPO}" --json headRefOid --jq '.headRefOid')"
+          fi
+          BODY="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/reviews" --paginate \
+            | jq -rs '
+                (add // [])
+                | map(select(.body != null and (.body | contains("<!-- ai-review-meta"))))
+                | sort_by(.submitted_at) | reverse
+                | .[0].body // empty
+              ')"
+          if [ -z "${BODY}" ]; then
+            echo "::error::No published review body containing <!-- ai-review-meta -->"
+            exit 1
+          fi
+          META="$(printf '%s\n' "${BODY}" | grep -E '<!--[[:space:]]*ai-review-meta' | head -n1 || true)"
+          echo "meta: ${META}"
+          echo "${META}" | grep -Eq 'head_sha=[0-9a-f]{40}'
+          echo "${META}" | grep -Eq 'base_sha=[0-9a-f]{40}'
+          echo "${META}" | grep -Eq 'mode=(full|delta|inconclusive)'
+          # Successful structured publishes stamp the PR tip; inconclusive may
+          # still carry head_sha for the next full-force. When verdict is set
+          # from Publish, expect the tip to match when mode is full|delta.
+          if [ "${VERDICT}" = "pass" ] || [ "${VERDICT}" = "fail" ]; then
+            if echo "${META}" | grep -Eq 'mode=(full|delta)'; then
+              echo "${META}" | grep -F "head_sha=${EXPECT_HEAD}"
+            fi
+          fi
+          echo "ai-review-meta OK (verdict=${VERDICT})"
+
   gated-demo:
     needs: [review]
     if: ${{ needs.review.outputs.verdict == 'pass' }}

README.md

@@ -71,6 +71,9 @@ owning organization.
   identity model, the composite-actions-vs-reusable-workflow choice, the
   parameterization surface, the public/generalized posture, and the
   supply-chain and least-privilege posture this repo follows.
+- [`docs/adr/0006-test-plan-ci-findings-and-osh-routing.md`](docs/adr/0006-test-plan-ci-findings-and-osh-routing.md)
+  — Test Plan ↔ CI findings (checklist ticks retired), delta reviews, and
+  roster-K Sonnet collapse / Opus fan-out routing.
 - [`docs/plan.md`](docs/plan.md) — the full build-out plan this repo is
   being implemented against, including the phase/task breakdown and the
   locked design decisions behind it.

ai-qa/README.md

@@ -77,7 +77,8 @@ prompt.)
 7. **Stage QA rubric** — copies the action's own `rubric.md` into the
    workspace so the review can read it with a stable path.
 8. **Post-merge QA review (agentic)** — only runs when an Anthropic
-   credential is configured. Claude (`qa-model`, Sonnet by default) reads the
+   credential is configured. Claude (locked Sonnet primary with
+   Cursor → free fallbacks) reads the
    rubric, inspects the merged diff via `git`, **smoke-tests the deployed app
    over HTTP** (the health URL plus any routes the diff touches), **evaluates
    the PR's Test Plan** if one is present (running each item it can against
@@ -114,7 +115,6 @@ prompt.)
 | `health-url` | URL polled with `curl --fail` until healthy or `deploy-timeout` elapses; also smoke-tested directly by the review. No sensible generic default exists. | **Yes** | — |
 | `deploy-timeout` | Seconds to keep polling `health-url` before giving up. | No | `180` |
 | `test-hint` | Optional free-text describing how to build/test this repo. Handed to the review as context — Claude MAY run it at its discretion to confirm a suspected regression, never mechanically. Consumer must provision the toolchain first. | No | `""` |
-| `qa-model` | Model used for the agentic QA review. | No | `claude-sonnet-5` |
 | `allowed-tools` | Tool allowlist passed to the review's `--allowedTools` (read/grep the code, `curl` the deploy, `git` the diff, optionally run a JS/TS build/test). Override to widen or narrow. | No | *(read/grep/glob + curl/git + node/npm/npx/yarn/pnpm/corepack)* |
 | `pass-label` | Label applied when the overall QA signal (health + review) passes. Also applied to linked issues when `update-linked-issues` is on. | No | `✓ /ai-qa` |
 | `fail-label` | Label applied when the overall QA signal fails. Also applied to linked issues (and the merge-auto-closed issue is reopened) when `update-linked-issues` is on. | No | `✗ /ai-qa` |

ai-qa/action.yml

@@ -52,12 +52,6 @@ inputs:
       before this action runs.
     required: false
     default: ""
-  qa-model:
-    description: >-
-      Model used for the agentic post-merge QA review. Defaults to Sonnet;
-      override with a repo var if a custom gateway aliases model names.
-    required: false
-    default: claude-sonnet-5
   allowed-tools:
     description: >-
       Tool allowlist passed to the QA review's `--allowedTools`. The default
@@ -292,6 +286,22 @@ runs:
       run: | # zizmor: ignore[github-env] BASE_URL is this action's own anthropic-base-url input (caller-supplied static config), never attacker-influenceable event data — see ADR-0010's env-binding rule
         echo "ANTHROPIC_BASE_URL=${BASE_URL}" >> "${GITHUB_ENV}"
 
+    # Locked model IDs (not inputs). Keep in sync with ai-review's *schema*
+    # cascade: Claude → structured_output free only (no Cursor — none advertise
+    # structured_output, and --json-schema reviews go inconclusive on them).
+    - name: Resolve locked models
+      id: models
+      if: >-
+        steps.merge.outputs.skip != 'true' &&
+        (inputs.anthropic-api-key != '' || inputs.anthropic-auth-token != '')
+      shell: bash
+      run: |
+        set -euo pipefail
+        {
+          echo "primary=claude/claude-sonnet-5"
+          echo "fallback=oc/nemotron-3.5-lightning-free,oc/deepseek-v4-flash-free,auto/best-free"
+        } >> "${GITHUB_OUTPUT}"
+
     # Workaround for anthropics/claude-code-action#1294: the action requires
     # *some* value in `anthropic_api_key` even when authenticating to a custom
     # gateway via a bearer token, so fall back to the auth token itself and
@@ -308,19 +318,14 @@ runs:
       with:
         anthropic_api_key: ${{ inputs.anthropic-api-key != '' && inputs.anthropic-api-key || inputs.anthropic-auth-token }}
         github_token: ${{ inputs.github-token }}
-        # --fallback-model: escape hatch for HTTP 529 (Overloaded). Without it
-        # the CLI retries the primary internally with no exit. Measured on the
-        # sibling ai-review action, that backoff ran ~28 min per stage and
-        # produced 85-min jobs that did zero work (turns:3, $0, 0 tool calls).
-        # This action has a single model stage, so the same overload costs it
-        # one long stall rather than three — but it had no exit either.
-        # Neither entry may equal the primary or the fallback is a no-op:
-        # `qa-model` defaults to Sonnet 5, so the list is same-tier first
-        # (Sonnet 4.6), then Opus. The list is ordered and the primary is
-        # re-tried at the start of each turn.
+        # --fallback-model: Claude primary → Cursor if Claude blocked → free
+        # if Cursor blocked. Keep the free tail in sync with ai-review.
+        # Neither entry may equal the primary or the fallback is a no-op.
+        # The list is ordered and the primary is re-tried at the start of
+        # each turn. Does not bound silent gateway stalls (ADR 0005).
         claude_args: |
-          --model ${{ inputs.qa-model }}
-          --fallback-model claude-sonnet-4-6,claude-opus-4-8
+          --model ${{ steps.models.outputs.primary }}
+          --fallback-model ${{ steps.models.outputs.fallback }}
           --allowedTools "${{ inputs.allowed-tools }}"
           --json-schema '{"type":"object","additionalProperties":false,"required":["verdict","confidence","merge_risk","deploy_status","counts","summary","report_markdown"],"properties":{"verdict":{"type":"string","enum":["pass","fail"]},"confidence":{"type":"integer","minimum":0,"maximum":100},"merge_risk":{"type":"string","enum":["low","medium","high"]},"deploy_status":{"type":"string","enum":["healthy","unhealthy","unknown"]},"counts":{"type":"object","additionalProperties":false,"required":["p0","p1","p2","p3"],"properties":{"p0":{"type":"integer","minimum":0},"p1":{"type":"integer","minimum":0},"p2":{"type":"integer","minimum":0},"p3":{"type":"integer","minimum":0}}},"summary":{"type":"string"},"report_markdown":{"type":"string"},"test_plan":{"type":"object","additionalProperties":false,"required":["present"],"properties":{"present":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["text","status"],"properties":{"text":{"type":"string"},"status":{"type":"string","enum":["passed","failed","unverifiable"]},"evidence":{"type":"string"}}}}}}}}'
         prompt: |
@@ -395,9 +400,33 @@ runs:
         UPDATE_PR_BODY: ${{ inputs.update-pr-body }}
         UPDATE_LINKED_ISSUES: ${{ inputs.update-linked-issues }}
         RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        REVIEW_EXEC_FILE: ${{ steps.review.outputs.execution_file }}
+        REPORT_FOOTER_PATH: ${{ github.action_path }}/lib/report-footer.js
+        # Same locked primary the review stage was started with (steps.models).
+        # Prefer the execution-log model when present; otherwise the requested
+        # primary — never a second hardcoded literal that can drift from --model.
+        QA_PRIMARY_MODEL: ${{ steps.models.outputs.primary }}
       with:
         github-token: ${{ steps.identity.outputs.author-token }}
         script: |
+          const fs = require('fs');
+          const { formatModelFooter, modelFromExecutionLog } = require(process.env.REPORT_FOOTER_PATH);
+
+          let modelUsed = '';
+          try {
+            const log = JSON.parse(fs.readFileSync(process.env.REVIEW_EXEC_FILE || '', 'utf8'));
+            modelUsed = modelFromExecutionLog(log);
+          } catch {
+            // No execution log (review skipped / missing).
+          }
+          // Only attribute a model when we have structured QA output. Prefer the
+          // log's resolved model (captures Cursor/free fallback). If the log is
+          // silent, fall back to the locked primary that this run requested —
+          // same shape as ai-review's resolveModelUsed({ fallback: ROUTED_MODEL }).
+          if (!modelUsed && process.env.QA_OUTPUT) {
+            modelUsed = process.env.QA_PRIMARY_MODEL || '';
+          }
+
           const prNumber = Number(process.env.PR_NUMBER);
           const healthOk = process.env.HEALTH_STATUS === 'healthy';
 
@@ -515,6 +544,12 @@ runs:
             body.push('> ⚠️ The QA review step reported a non-zero outcome — the report above may be partial.');
           }
 
+          const footer = formatModelFooter(modelUsed);
+          if (footer) {
+            body.push('');
+            body.push(footer);
+          }
+
           body.push('');
           body.push(`_Posted by \`${process.env.AUTHOR_LOGIN}\` · [workflow run](${process.env.RUN_URL})_`);
 

ai-qa/lib/report-footer.js

@@ -0,0 +1,28 @@
+"use strict";
+
+/**
+ * @param {string|null|undefined} modelUsed
+ * @returns {string} empty when modelUsed is blank
+ */
+function formatModelFooter(modelUsed) {
+  if (!modelUsed || typeof modelUsed !== "string" || !modelUsed.trim()) return "";
+  return [
+    `Model: \`${modelUsed.trim()}\``,
+    "_Re-run this job if you need another review pass._",
+  ].join("\n");
+}
+
+/**
+ * First named model in a claude-code-action execution log.
+ * @param {unknown} entries
+ * @returns {string}
+ */
+function modelFromExecutionLog(entries) {
+  if (!Array.isArray(entries)) return "";
+  for (const e of entries) {
+    if (e && typeof e.model === "string" && e.model.trim()) return e.model.trim();
+  }
+  return "";
+}
+
+module.exports = { formatModelFooter, modelFromExecutionLog };

ai-qa/lib/report-footer.test.js

@@ -0,0 +1,27 @@
+"use strict";
+
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { formatModelFooter, modelFromExecutionLog } = require("./report-footer.js");
+
+test("formatModelFooter renders model and re-run hint", () => {
+  const s = formatModelFooter("claude/claude-sonnet-5");
+  assert.match(s, /Model: `claude\/claude-sonnet-5`/);
+  assert.match(s, /Re-run this job if you need another review pass/);
+});
+
+test("formatModelFooter returns empty for blank", () => {
+  assert.equal(formatModelFooter(""), "");
+  assert.equal(formatModelFooter(null), "");
+});
+
+test("modelFromExecutionLog reads the first named model", () => {
+  assert.equal(
+    modelFromExecutionLog([
+      { type: "system", subtype: "init", model: "oc/mimo-v2.5-free" },
+      { type: "result", subtype: "success" },
+    ]),
+    "oc/mimo-v2.5-free",
+  );
+  assert.equal(modelFromExecutionLog(null), "");
+});