From 9a1ea31bc94629cde109ebc9f75963a375a06681 Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 07:15:53 +0300 Subject: [PATCH 01/10] feat(ai-review): show model used and re-run hint on review comments Refs #51 --- ai-review/lib/publish.js | 25 ++++++++++++++++++++++--- ai-review/lib/publish.test.js | 23 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/ai-review/lib/publish.js b/ai-review/lib/publish.js index df279a6..a9f1f89 100644 --- a/ai-review/lib/publish.js +++ b/ai-review/lib/publish.js @@ -45,9 +45,21 @@ function stripLeadingBannerArtifacts(markdown) { /** * @param {{verdict: string, confidence: number, mergeRisk: string, * counts: {p0:number,p1:number,p2:number,p3:number}, intentDeviated: boolean, - * modelVerdict: string|undefined, blockers: string[], commentBody: string}} args + * modelVerdict: string|undefined, blockers: string[], commentBody: string, + * modelUsed?: string|null}} args * `commentBody` must already be run through stripLeadingBannerArtifacts. */ +function modelLine(modelUsed) { + if (!modelUsed || typeof modelUsed !== "string" || !modelUsed.trim()) return []; + return ["", `Model: \`${modelUsed.trim()}\``]; +} + +function modelFooter(modelUsed) { + const line = modelLine(modelUsed); + if (!line.length) return []; + return [...line, "_Re-run this job if you need another review pass._"]; +} + function buildReviewBody({ verdict, confidence, @@ -57,6 +69,7 @@ function buildReviewBody({ modelVerdict, blockers, commentBody, + modelUsed, }) { const verdictLine = verdict === "pass" ? "**✅ PASS**" : "**❌ FAIL**"; const rejectedBanner = intentDeviated ? "❌ **Rejected — wrong solution**\n\n" : ""; @@ -91,13 +104,18 @@ function buildReviewBody({ "", `Confidence: ${confidence} · Merge risk: ${mergeRisk}`, `P0: ${counts.p0} · P1: ${counts.p1} · P2: ${counts.p2} · P3: ${counts.p3}`, + ...modelFooter(modelUsed), "", commentBody || "_No review content returned._", ].join("\n"); } -/** @param {string} salvaged possibly-empty text recovered from a missed structured output. */ -function buildInconclusiveBody(salvaged) { +/** + * @param {string} salvaged possibly-empty text recovered from a missed structured output. + * @param {{modelUsed?: string|null}} [opts] + */ +function buildInconclusiveBody(salvaged, opts = {}) { + const modelUsed = opts && opts.modelUsed; return [ "", "### ⚠️ AI Review — inconclusive (re-run required)", @@ -109,6 +127,7 @@ function buildInconclusiveBody(salvaged) { "fails closed.", "", "**Re-run the `ai-review` job** to get a verdict.", + ...modelLine(modelUsed), ...(salvaged ? [ "", diff --git a/ai-review/lib/publish.test.js b/ai-review/lib/publish.test.js index 4b23307..eea4f8f 100644 --- a/ai-review/lib/publish.test.js +++ b/ai-review/lib/publish.test.js @@ -121,6 +121,20 @@ test("always leads with the marker", () => { assert.match(body, /^\n/); }); +test("buildReviewBody includes model used and re-review hint", () => { + const body = buildReviewBody({ + ...BASE_ARGS, + modelUsed: "claude/claude-sonnet-5", + }); + assert.match(body, /Model: `claude\/claude-sonnet-5`/); + assert.match(body, /Re-run this job if you need another review pass/); +}); + +test("buildReviewBody omits Model line when modelUsed is empty", () => { + const body = buildReviewBody({ ...BASE_ARGS, modelUsed: "" }); + assert.doesNotMatch(body, /^Model:/m); +}); + // --- buildInconclusiveBody --------------------------------------------------- test("without salvaged text there is no details block", () => { @@ -136,6 +150,15 @@ test("with salvaged text the details block contains it", () => { assert.match(body, /<\/details>/); }); +test("buildInconclusiveBody includes Model line but not a second italic re-run hint", () => { + const body = buildInconclusiveBody("salvaged text", { + modelUsed: "claude/cursor/composer-2.5", + }); + assert.match(body, /Model: `claude\/cursor\/composer-2.5`/); + assert.match(body, /\*\*Re-run the `ai-review` job\*\*/); + assert.doesNotMatch(body, /_Re-run this job if you need another review pass\._/); +}); + // --- tickVerifiedBoxes -------------------------------------------------------- test("ticks a single matching unchecked box", () => { From 68a2d98a37b226b905a1cbb72e99afd8a252cd82 Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 07:17:50 +0300 Subject: [PATCH 02/10] feat(ai-review): attribute PR review to the model that actually ran Refs #51 --- ai-review/action.yml | 25 ++++++++++++++++++++++++- ai-review/lib/metrics.js | 25 ++++++++++++++++++++++++- ai-review/lib/metrics.test.js | 20 +++++++++++++++++++- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/ai-review/action.yml b/ai-review/action.yml index 17af413..ddea172 100644 --- a/ai-review/action.yml +++ b/ai-review/action.yml @@ -1515,6 +1515,11 @@ runs: || steps.review.outputs.structured_output) }} RECOMPUTE_PATH: ${{ github.action_path }}/lib/recompute.js PUBLISH_LIB_PATH: ${{ github.action_path }}/lib/publish.js + METRICS_PATH: ${{ github.action_path }}/lib/metrics.js + ROUTED_MODEL: ${{ steps.route.outputs.model }} + REVIEW_LOG: ${{ runner.temp }}/ai-review-exec-review-snapshot.json + REPAIR_LOG: ${{ runner.temp }}/ai-review-exec-repair-snapshot.json + RETRY_LOG: ${{ steps.review_retry.outputs.execution_file }} with: github-token: ${{ steps.identity-refresh.outputs.author-token }} script: | @@ -1528,6 +1533,23 @@ runs: buildStatusBlock, upsertStatusBlock, } = require(process.env.PUBLISH_LIB_PATH); + const { resolveModelUsed } = require(process.env.METRICS_PATH); + + const readLog = (p) => { + try { + return JSON.parse(fs.readFileSync(p, "utf8")); + } catch { + return null; + } + }; + const modelUsed = resolveModelUsed({ + logs: [ + readLog(process.env.RETRY_LOG), + readLog(process.env.REPAIR_LOG), + readLog(process.env.REVIEW_LOG), + ], + fallback: process.env.ROUTED_MODEL || "", + }); const prNumber = Number(process.env.PR_NUMBER); const passLabel = process.env.PASS_LABEL; @@ -1552,7 +1574,7 @@ runs: } catch (e) { // No salvage file (step skipped, or nothing recoverable). } - const inconclusiveBody = buildInconclusiveBody(salvaged); + const inconclusiveBody = buildInconclusiveBody(salvaged, { modelUsed }); try { await github.rest.pulls.createReview({ owner: context.repo.owner, @@ -1624,6 +1646,7 @@ runs: modelVerdict: review.verdict, blockers, commentBody, + modelUsed, }); await github.rest.pulls.createReview({ diff --git a/ai-review/lib/metrics.js b/ai-review/lib/metrics.js index de44c59..99be815 100644 --- a/ai-review/lib/metrics.js +++ b/ai-review/lib/metrics.js @@ -207,4 +207,27 @@ function renderSummary(metrics) { ); } -module.exports = { parseExecutionLog, collectMetrics, renderSummary, formatDuration, isStalled, STALL_MIN_MS }; +/** + * Prefer the first execution log that names a model (retry → repair → review + * order is the caller's responsibility). Fall back to the routed primary when + * logs are missing or silent. + * @param {{logs: unknown[], fallback: string}} args + * @returns {string} + */ +function resolveModelUsed({ logs, fallback }) { + for (const log of logs || []) { + const m = parseExecutionLog(log).model; + if (typeof m === "string" && m.trim()) return m.trim(); + } + return typeof fallback === "string" ? fallback : ""; +} + +module.exports = { + parseExecutionLog, + collectMetrics, + renderSummary, + formatDuration, + isStalled, + STALL_MIN_MS, + resolveModelUsed, +}; diff --git a/ai-review/lib/metrics.test.js b/ai-review/lib/metrics.test.js index 9210341..5d339c4 100644 --- a/ai-review/lib/metrics.test.js +++ b/ai-review/lib/metrics.test.js @@ -3,7 +3,7 @@ const test = require("node:test"); const assert = require("node:assert/strict"); -const { parseExecutionLog, collectMetrics, renderSummary } = require("./metrics.js"); +const { parseExecutionLog, collectMetrics, renderSummary, resolveModelUsed } = require("./metrics.js"); // A minimal execution log in claude-code-action's shape: a top-level array of // stream entries, terminated by a `result` entry. The shape is pinned by the @@ -236,3 +236,21 @@ test("renderSummary surfaces a stall rather than burying it in the table", () => assert.match(md, /stall/i); assert.match(md, /review/); }); + +// --- resolveModelUsed ------------------------------------------------------ + +test("resolveModelUsed prefers first log that names a model", () => { + const retry = logFor({ turns: 1, cost: 0, ms: 10, model: "oc/mimo-v2.5-free" }); + const review = logFor({ turns: 1, cost: 0, ms: 10, model: "claude/claude-opus-5" }); + assert.equal( + resolveModelUsed({ logs: [retry, review], fallback: "claude/claude-opus-5" }), + "oc/mimo-v2.5-free", + ); +}); + +test("resolveModelUsed falls back when logs empty", () => { + assert.equal( + resolveModelUsed({ logs: [null, []], fallback: "claude/claude-sonnet-5" }), + "claude/claude-sonnet-5", + ); +}); From ebaceeb0669e5df311afbb0c00b2e85e9cdfb957 Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 07:19:31 +0300 Subject: [PATCH 03/10] feat(ai-review): lock Claude primaries with Cursor then free fallbacks Refs #51 --- ai-review/README.md | 12 +++--- ai-review/action.yml | 69 +++++++++++++-------------------- ai-review/lib/write-manifest.js | 6 +-- 3 files changed, 35 insertions(+), 52 deletions(-) diff --git a/ai-review/README.md b/ai-review/README.md index 2475487..aea28d4 100644 --- a/ai-review/README.md +++ b/ai-review/README.md @@ -59,8 +59,9 @@ injection-safety rule. derives the diff base from a false premise reviews the wrong range and reports confidently on it. - The same step routes ordinary diffs to `sonnet-model`, escalating to - `opus-model` once a diff exceeds **either** `sonnet-files-threshold` + The same step routes ordinary diffs to the locked Sonnet primary + (`claude/claude-sonnet-5`), escalating to Opus (`claude/claude-opus-5`) + once a diff exceeds **either** `sonnet-files-threshold` (25) or `sonnet-churn-threshold` (800). These were briefly 3/60, which sent nearly every real PR to Opus and moved the review stage from ~10-13 min to a 35-min median. Widened 15/400 → 25/800 after measuring @@ -204,11 +205,8 @@ injection-safety rule. | `qa-pass-label` | Post-merge `ai-qa` pass label; cleared (not applied) by this action on every new commit. | No | `✓ /ai-qa` | | `qa-fail-label` | Post-merge `ai-qa` fail label; cleared (not applied) by this action on every new commit. | No | `✗ /ai-qa` | | `confidence-threshold` | Minimum **blocking-finding** confidence (0-100) required for a pass. The Publish step recomputes confidence from the review stage's P0/P1 counts and test-quality signals and compares it against this threshold. P2/P3 findings lower the *reported* confidence but are advisory and never block. | No | `90` | -| `sonnet-files-threshold` | Max changed-file count for a diff to still route to `sonnet-model` (must hold together with `sonnet-churn-threshold`); larger diffs route to `opus-model`. | No | `25` | -| `sonnet-churn-threshold` | Max changed-line count (adds + deletes) for a diff to still route to `sonnet-model`. | No | `800` | -| `sonnet-model` | Model the routing step selects for diffs within **both** thresholds. Override when a gateway aliases model names. | No | `claude-sonnet-5` | -| `opus-model` | Model the routing step selects for every larger diff. Override when a gateway aliases model names. | No | `claude-opus-5` | -| `haiku-model` | Model used by the context stage, and stamped on the roster's `history`/`scorer` roles by the prep step. Note: Haiku 4.5 does not accept the `effort` parameter, so no stage or role running it passes `--effort` — the roster resolves that against the model id, not the tier, so overriding another tier to a literal Haiku id (e.g. `sonnet-model: claude-haiku-4-5`) is also covered. A gateway alias that *routes* to Haiku under an unrelated string is not detected; the check is a substring match, not alias resolution. | No | `claude-haiku-4-5` | +| `sonnet-files-threshold` | Max changed-file count for a diff to still route to the locked Sonnet primary (must hold together with `sonnet-churn-threshold`); larger diffs route to Opus. | No | `25` | +| `sonnet-churn-threshold` | Max changed-line count (adds + deletes) for a diff to still route to Sonnet. | No | `800` | | `enable-context-stage` | When `false`, skips the Haiku context stage (and its `context.md` verification) entirely. The stage is best-effort and its output optional, so disabling it removes a wall-clock risk without changing the gate contract. | No | `true` | | `api-timeout-ms` | Per-request timeout (ms) for every Claude stage, passed as `API_TIMEOUT_MS` (CLI default `600000`). **Does not bound the ~27.5-min stall** — a run with this set to `180000` still stalled 27m36s. It is a genuine per-request bound and fails a wedged request faster than the default, nothing more. | No | `180000` | | `test-command` | **DEPRECATED — accepted but ignored.** The Review stage no longer runs tests; see [Why the review no longer runs tests](#why-the-review-no-longer-runs-tests). | No | — | diff --git a/ai-review/action.yml b/ai-review/action.yml index ddea172..7afdcf0 100644 --- a/ai-review/action.yml +++ b/ai-review/action.yml @@ -94,30 +94,12 @@ inputs: docs/adr/0004-non-blocking-findings-and-structured-output-repair.md. required: false default: "90" - sonnet-model: - description: > - Model ID used by the diff-size routing step ONLY for tiny diffs - (≤`sonnet-files-threshold` files AND ≤`sonnet-churn-threshold` - changed lines). Every larger diff routes to `opus-model`. Consumed - by the routing step, whose output feeds the Review stage as the - model to run. Override when a gateway aliases model names. - required: false - default: "claude-sonnet-5" - opus-model: - description: > - Model ID used by the diff-size routing step for all non-tiny diffs - (the default — the review stage prefers Opus, dropping to - `sonnet-model` only for trivially small changes). Consumed by the - routing step, whose output feeds the Review stage as the model to - run. Override when a gateway aliases model names. - required: false - default: "claude-opus-5" sonnet-files-threshold: description: > Max changed-file count for a diff to still be considered "tiny" and - routed to `sonnet-model`. A diff qualifies for Sonnet only when it + routed to the locked Sonnet primary. A diff qualifies for Sonnet only when it is at or under BOTH this and `sonnet-churn-threshold`; anything - larger routes to `opus-model`. Widened to 25 (from 15) — measured over + larger routes to the locked Opus primary. Widened to 25 (from 15) — measured over 682 review jobs, 89% of all traffic routed to Opus, and Opus runs cost 3x the wall-clock (28.1 min median vs 9.2) and 4x the spend ($13.22 vs $3.27) of Sonnet ones. This is NOT a depth concession: the parallel @@ -129,23 +111,11 @@ inputs: sonnet-churn-threshold: description: > Max changed-line count (additions + deletions) for a diff to still - be considered "tiny" and routed to `sonnet-model`. Applied together + be considered "tiny" and routed to the locked Sonnet primary. Applied together with `sonnet-files-threshold` (both must hold for Sonnet). Widened to 800 alongside `sonnet-files-threshold` — see the note there. required: false default: "800" - haiku-model: - description: > - Model ID used by the context stage (this phase) to read the diff and changed - files and write context.md for the review stage to consume. Also stamped by - the deterministic-prep step onto the roster's `history` and `scorer` roles, - so an override here reaches those too. Override when a gateway aliases model - names. NOTE: Haiku 4.5 does not accept the `effort` parameter, so no stage - running this model may pass `--effort` — the roster resolves that against - the model id rather than the tier, so pointing another tier at a Haiku id is - covered as well. - required: false - default: "claude-haiku-4-5" enable-context-stage: description: > When 'true' (default) the Haiku context stage runs and writes @@ -438,12 +408,8 @@ runs: shell: bash env: BASE_REF: ${{ steps.pr-state.outputs.base-ref }} - SONNET: ${{ inputs.sonnet-model }} - OPUS: ${{ inputs.opus-model }} - # Read by lib/write-manifest.js only (lib/roster.js is pure and never - # touches the environment), to stamp the OSH tiering onto the roster - # with the consumer's overrides rather than hardcoded ids. - HAIKU: ${{ inputs.haiku-model }} + # Locked at action level — keep in sync with ai-qa cascade comments. + # Claude primary → Cursor if Claude blocked → free if Cursor blocked. FILES_MAX: ${{ inputs.sonnet-files-threshold }} CHURN_MAX: ${{ inputs.sonnet-churn-threshold }} MANIFEST_CLI: ${{ github.action_path }}/lib/write-manifest.js @@ -454,6 +420,16 @@ runs: set -euo pipefail mkdir -p .ai-review + # Locked model IDs (not inputs). Keep in sync with ai-qa. + SONNET="claude/claude-sonnet-5" + OPUS="claude/claude-opus-5" + HAIKU="claude/claude-haiku-4-5-20251001" + FREE="oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free" + SONNET_FALLBACK="claude/cursor/claude-4.6-sonnet-medium-thinking,${FREE}" + OPUS_FALLBACK="claude/cursor/claude-opus-4-8-medium-fast,${FREE}" + HAIKU_FALLBACK="claude/cursor/composer-2.5,${FREE}" + export SONNET OPUS HAIKU + # Resolve the review range once, deterministically. `git diff A...B` # already means "merge base of A and B", but resolving the SHA # explicitly puts it in the manifest and the job log, so a wrong-range @@ -499,14 +475,19 @@ runs: FILES_MAX="${FILES_MAX:-25}" CHURN_MAX="${CHURN_MAX:-800}" if [ "${FILES}" -le "${FILES_MAX}" ] && [ "${CHURN}" -le "${CHURN_MAX}" ]; then - MODEL="${SONNET:-claude-sonnet-5}" + MODEL="${SONNET}" + FALLBACK="${SONNET_FALLBACK}" else - MODEL="${OPUS:-claude-opus-5}" + MODEL="${OPUS}" + FALLBACK="${OPUS_FALLBACK}" fi { echo "files=${FILES}" echo "churn=${CHURN}" echo "model=${MODEL}" + echo "fallback-model=${FALLBACK}" + echo "haiku-model=${HAIKU}" + echo "haiku-fallback-model=${HAIKU_FALLBACK}" } >> "${GITHUB_OUTPUT}" - name: Resolve linked issues @@ -586,7 +567,8 @@ runs: # Review stage's relative-path read of `context.md` at the repository root # resolves to the same file. claude_args: | - --model ${{ inputs.haiku-model }} + --model ${{ steps.route.outputs.haiku-model }} + --fallback-model ${{ steps.route.outputs.haiku-fallback-model }} --allowedTools "Read,Grep,Glob,Write,Bash(git diff:*),Bash(git symbolic-ref:*),Bash(git remote show:*),Bash(git merge-base:*),Bash(git log:*),Bash(git show:*)" prompt: | Read `.ai-review/manifest.json` first: a prior deterministic step @@ -748,6 +730,7 @@ runs: plugin_marketplaces: https://github.com/obra/superpowers-marketplace.git claude_args: | --model ${{ steps.route.outputs.model }} + --fallback-model ${{ steps.route.outputs.fallback-model }} --max-turns 200 --allowedTools "Read,Grep,Glob,Bash(gh pr view:*),Bash(gh issue view:*),Bash(git diff:*),Bash(git symbolic-ref:*),Bash(git merge-base:*),Bash(git remote show:*),Bash(git log:*),Bash(git show:*),Bash(cat:*),Bash(head:*),Bash(tail:*),Bash(ls:*),Bash(find:*),Bash(wc:*)" --json-schema '{"type":"object","additionalProperties":false,"required":["verdict","confidence","merge_risk","intent","counts","review_event","comment_markdown"],"properties":{"verdict":{"type":"string","enum":["pass","fail"]},"confidence":{"type":"integer","minimum":0,"maximum":100},"merge_risk":{"type":"string","enum":["low","med","high"]},"intent":{"type":"string","enum":["aligned","partial","deviated","skipped"]},"counts":{"type":"object","additionalProperties":false,"required":["p0","p1","p2","p3"],"properties":{"p0":{"type":"integer","minimum":0},"p1":{"type":"integer","minimum":0},"p2":{"type":"integer","minimum":0},"p3":{"type":"integer","minimum":0}}},"review_event":{"type":"string","enum":["APPROVE","REQUEST_CHANGES"]},"comment_markdown":{"type":"string"},"findings":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["id","file","line","severity","summary","failure_scenario","reason","evidence","confidence"],"properties":{"id":{"type":"string"},"file":{"type":"string"},"line":{"type":"integer","minimum":0},"severity":{"type":"string","enum":["P0","P1","P2","P3"]},"summary":{"type":"string"},"failure_scenario":{"type":"string"},"reason":{"type":"string"},"evidence":{"type":"string"},"confidence":{"type":"integer","enum":[0,25,50,75,100]},"severity_confirmed":{"type":"string","enum":["P0","P1","P2","P3"]}}}},"files_reviewed":{"type":"array","items":{"type":"string"}},"tests_failing":{"type":"boolean"},"coverage_below_threshold_on_critical_paths":{"type":"boolean"},"no_tests_for_changed_logic":{"type":"boolean"},"test_execution":{"type":"string","enum":["passed","failed","skipped","not_run"]},"verification_evidence":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["claim","command","result"],"properties":{"claim":{"type":"string"},"command":{"type":"string"},"result":{"type":"string"}}}},"checklist":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["text","status"],"properties":{"text":{"type":"string"},"status":{"type":"string","enum":["verified","failed","unverifiable"]},"evidence":{"type":"string"}}}}}}' @@ -957,6 +940,7 @@ runs: claude_args: | --resume ${{ steps.review.outputs.session_id }} --model ${{ steps.route.outputs.model }} + --fallback-model ${{ steps.route.outputs.fallback-model }} --allowedTools "Read" --json-schema '{"type":"object","additionalProperties":false,"required":["verdict","confidence","merge_risk","intent","counts","review_event","comment_markdown"],"properties":{"verdict":{"type":"string","enum":["pass","fail"]},"confidence":{"type":"integer","minimum":0,"maximum":100},"merge_risk":{"type":"string","enum":["low","med","high"]},"intent":{"type":"string","enum":["aligned","partial","deviated","skipped"]},"counts":{"type":"object","additionalProperties":false,"required":["p0","p1","p2","p3"],"properties":{"p0":{"type":"integer","minimum":0},"p1":{"type":"integer","minimum":0},"p2":{"type":"integer","minimum":0},"p3":{"type":"integer","minimum":0}}},"review_event":{"type":"string","enum":["APPROVE","REQUEST_CHANGES"]},"comment_markdown":{"type":"string"},"findings":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["id","file","line","severity","summary","failure_scenario","reason","evidence","confidence"],"properties":{"id":{"type":"string"},"file":{"type":"string"},"line":{"type":"integer","minimum":0},"severity":{"type":"string","enum":["P0","P1","P2","P3"]},"summary":{"type":"string"},"failure_scenario":{"type":"string"},"reason":{"type":"string"},"evidence":{"type":"string"},"confidence":{"type":"integer","enum":[0,25,50,75,100]},"severity_confirmed":{"type":"string","enum":["P0","P1","P2","P3"]}}}},"files_reviewed":{"type":"array","items":{"type":"string"}},"tests_failing":{"type":"boolean"},"coverage_below_threshold_on_critical_paths":{"type":"boolean"},"no_tests_for_changed_logic":{"type":"boolean"},"test_execution":{"type":"string","enum":["passed","failed","skipped","not_run"]},"verification_evidence":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["claim","command","result"],"properties":{"claim":{"type":"string"},"command":{"type":"string"},"result":{"type":"string"}}}},"checklist":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["text","status"],"properties":{"text":{"type":"string"},"status":{"type":"string","enum":["verified","failed","unverifiable"]},"evidence":{"type":"string"}}}}}}' prompt: | @@ -1045,6 +1029,7 @@ runs: plugin_marketplaces: https://github.com/obra/superpowers-marketplace.git claude_args: | --model ${{ steps.route.outputs.model }} + --fallback-model ${{ steps.route.outputs.fallback-model }} --max-turns 200 --allowedTools "Read,Grep,Glob,Bash(gh pr view:*),Bash(gh issue view:*),Bash(git diff:*),Bash(git symbolic-ref:*),Bash(git merge-base:*),Bash(git remote show:*),Bash(git log:*),Bash(git show:*),Bash(cat:*),Bash(head:*),Bash(tail:*),Bash(ls:*),Bash(find:*),Bash(wc:*)" --json-schema '{"type":"object","additionalProperties":false,"required":["verdict","confidence","merge_risk","intent","counts","review_event","comment_markdown"],"properties":{"verdict":{"type":"string","enum":["pass","fail"]},"confidence":{"type":"integer","minimum":0,"maximum":100},"merge_risk":{"type":"string","enum":["low","med","high"]},"intent":{"type":"string","enum":["aligned","partial","deviated","skipped"]},"counts":{"type":"object","additionalProperties":false,"required":["p0","p1","p2","p3"],"properties":{"p0":{"type":"integer","minimum":0},"p1":{"type":"integer","minimum":0},"p2":{"type":"integer","minimum":0},"p3":{"type":"integer","minimum":0}}},"review_event":{"type":"string","enum":["APPROVE","REQUEST_CHANGES"]},"comment_markdown":{"type":"string"},"findings":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["id","file","line","severity","summary","failure_scenario","reason","evidence","confidence"],"properties":{"id":{"type":"string"},"file":{"type":"string"},"line":{"type":"integer","minimum":0},"severity":{"type":"string","enum":["P0","P1","P2","P3"]},"summary":{"type":"string"},"failure_scenario":{"type":"string"},"reason":{"type":"string"},"evidence":{"type":"string"},"confidence":{"type":"integer","enum":[0,25,50,75,100]},"severity_confirmed":{"type":"string","enum":["P0","P1","P2","P3"]}}}},"files_reviewed":{"type":"array","items":{"type":"string"}},"tests_failing":{"type":"boolean"},"coverage_below_threshold_on_critical_paths":{"type":"boolean"},"no_tests_for_changed_logic":{"type":"boolean"},"test_execution":{"type":"string","enum":["passed","failed","skipped","not_run"]},"verification_evidence":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["claim","command","result"],"properties":{"claim":{"type":"string"},"command":{"type":"string"},"result":{"type":"string"}}}},"checklist":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["text","status"],"properties":{"text":{"type":"string"},"status":{"type":"string","enum":["verified","failed","unverifiable"]},"evidence":{"type":"string"}}}}}}' diff --git a/ai-review/lib/write-manifest.js b/ai-review/lib/write-manifest.js index c704471..0fdcc22 100644 --- a/ai-review/lib/write-manifest.js +++ b/ai-review/lib/write-manifest.js @@ -182,9 +182,9 @@ function writeRoster(manifest, sizes, io) { roster = buildRoster({ files: manifest.changed_files.map((p) => ({ path: p, bytes: sizes[p] ?? 0 })), models: { - opus: process.env.OPUS || "claude-opus-5", - sonnet: process.env.SONNET || "claude-sonnet-5", - haiku: process.env.HAIKU || "claude-haiku-4-5", + opus: process.env.OPUS || "claude/claude-opus-5", + sonnet: process.env.SONNET || "claude/claude-sonnet-5", + haiku: process.env.HAIKU || "claude/claude-haiku-4-5-20251001", }, importEdges: resolveImportEdges(specifiers, manifest.changed_files), symbolManifest: manifest.symbol_manifest, From 05eeb190f4010454f91b5f0fa15bf0e64a45463a Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 07:20:11 +0300 Subject: [PATCH 04/10] feat(ai-qa): lock Claude primary with Cursor then free fallbacks Refs #51 --- ai-qa/README.md | 3 ++- ai-qa/action.yml | 47 ++++++++++++++++++++------------- ai-qa/lib/report-footer.js | 28 ++++++++++++++++++++ ai-qa/lib/report-footer.test.js | 27 +++++++++++++++++++ 4 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 ai-qa/lib/report-footer.js create mode 100644 ai-qa/lib/report-footer.test.js diff --git a/ai-qa/README.md b/ai-qa/README.md index 1855bad..6c7034d 100644 --- a/ai-qa/README.md +++ b/ai-qa/README.md @@ -77,7 +77,8 @@ prompt.) 7. **Stage QA rubric** — copies the action's own `rubric.md` into the workspace so the review can read it with a stable path. 8. **Post-merge QA review (agentic)** — only runs when an Anthropic - credential is configured. Claude (`qa-model`, Sonnet by default) reads the + credential is configured. Claude (locked Sonnet primary with + Cursor → free fallbacks) reads the rubric, inspects the merged diff via `git`, **smoke-tests the deployed app over HTTP** (the health URL plus any routes the diff touches), **evaluates the PR's Test Plan** if one is present (running each item it can against diff --git a/ai-qa/action.yml b/ai-qa/action.yml index 7f9290e..a8a0a36 100644 --- a/ai-qa/action.yml +++ b/ai-qa/action.yml @@ -52,12 +52,6 @@ inputs: before this action runs. required: false default: "" - qa-model: - description: >- - Model used for the agentic post-merge QA review. Defaults to Sonnet; - override with a repo var if a custom gateway aliases model names. - required: false - default: claude-sonnet-5 allowed-tools: description: >- Tool allowlist passed to the QA review's `--allowedTools`. The default @@ -308,19 +302,14 @@ runs: with: anthropic_api_key: ${{ inputs.anthropic-api-key != '' && inputs.anthropic-api-key || inputs.anthropic-auth-token }} github_token: ${{ inputs.github-token }} - # --fallback-model: escape hatch for HTTP 529 (Overloaded). Without it - # the CLI retries the primary internally with no exit. Measured on the - # sibling ai-review action, that backoff ran ~28 min per stage and - # produced 85-min jobs that did zero work (turns:3, $0, 0 tool calls). - # This action has a single model stage, so the same overload costs it - # one long stall rather than three — but it had no exit either. - # Neither entry may equal the primary or the fallback is a no-op: - # `qa-model` defaults to Sonnet 5, so the list is same-tier first - # (Sonnet 4.6), then Opus. The list is ordered and the primary is - # re-tried at the start of each turn. + # --fallback-model: Claude primary → Cursor if Claude blocked → free + # if Cursor blocked. Keep the free tail in sync with ai-review. + # Neither entry may equal the primary or the fallback is a no-op. + # The list is ordered and the primary is re-tried at the start of + # each turn. Does not bound silent gateway stalls (ADR 0005). claude_args: | - --model ${{ inputs.qa-model }} - --fallback-model claude-sonnet-4-6,claude-opus-4-8 + --model claude/claude-sonnet-5 + --fallback-model claude/cursor/claude-4.6-sonnet-medium-thinking,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free --allowedTools "${{ inputs.allowed-tools }}" --json-schema '{"type":"object","additionalProperties":false,"required":["verdict","confidence","merge_risk","deploy_status","counts","summary","report_markdown"],"properties":{"verdict":{"type":"string","enum":["pass","fail"]},"confidence":{"type":"integer","minimum":0,"maximum":100},"merge_risk":{"type":"string","enum":["low","medium","high"]},"deploy_status":{"type":"string","enum":["healthy","unhealthy","unknown"]},"counts":{"type":"object","additionalProperties":false,"required":["p0","p1","p2","p3"],"properties":{"p0":{"type":"integer","minimum":0},"p1":{"type":"integer","minimum":0},"p2":{"type":"integer","minimum":0},"p3":{"type":"integer","minimum":0}}},"summary":{"type":"string"},"report_markdown":{"type":"string"},"test_plan":{"type":"object","additionalProperties":false,"required":["present"],"properties":{"present":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["text","status"],"properties":{"text":{"type":"string"},"status":{"type":"string","enum":["passed","failed","unverifiable"]},"evidence":{"type":"string"}}}}}}}}' prompt: | @@ -395,9 +384,25 @@ runs: UPDATE_PR_BODY: ${{ inputs.update-pr-body }} UPDATE_LINKED_ISSUES: ${{ inputs.update-linked-issues }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REVIEW_EXEC_FILE: ${{ steps.review.outputs.execution_file }} + REPORT_FOOTER_PATH: ${{ github.action_path }}/lib/report-footer.js with: github-token: ${{ steps.identity.outputs.author-token }} script: | + const fs = require('fs'); + const { formatModelFooter, modelFromExecutionLog } = require(process.env.REPORT_FOOTER_PATH); + + let modelUsed = ''; + try { + const log = JSON.parse(fs.readFileSync(process.env.REVIEW_EXEC_FILE || '', 'utf8')); + modelUsed = modelFromExecutionLog(log); + } catch { + // No execution log (review skipped / missing). + } + if (!modelUsed && process.env.QA_OUTPUT) { + modelUsed = 'claude/claude-sonnet-5'; + } + const prNumber = Number(process.env.PR_NUMBER); const healthOk = process.env.HEALTH_STATUS === 'healthy'; @@ -515,6 +520,12 @@ runs: body.push('> ⚠️ The QA review step reported a non-zero outcome — the report above may be partial.'); } + const footer = formatModelFooter(modelUsed); + if (footer) { + body.push(''); + body.push(footer); + } + body.push(''); body.push(`_Posted by \`${process.env.AUTHOR_LOGIN}\` · [workflow run](${process.env.RUN_URL})_`); diff --git a/ai-qa/lib/report-footer.js b/ai-qa/lib/report-footer.js new file mode 100644 index 0000000..67c9583 --- /dev/null +++ b/ai-qa/lib/report-footer.js @@ -0,0 +1,28 @@ +"use strict"; + +/** + * @param {string|null|undefined} modelUsed + * @returns {string} empty when modelUsed is blank + */ +function formatModelFooter(modelUsed) { + if (!modelUsed || typeof modelUsed !== "string" || !modelUsed.trim()) return ""; + return [ + `Model: \`${modelUsed.trim()}\``, + "_Re-run this job if you need another review pass._", + ].join("\n"); +} + +/** + * First named model in a claude-code-action execution log. + * @param {unknown} entries + * @returns {string} + */ +function modelFromExecutionLog(entries) { + if (!Array.isArray(entries)) return ""; + for (const e of entries) { + if (e && typeof e.model === "string" && e.model.trim()) return e.model.trim(); + } + return ""; +} + +module.exports = { formatModelFooter, modelFromExecutionLog }; diff --git a/ai-qa/lib/report-footer.test.js b/ai-qa/lib/report-footer.test.js new file mode 100644 index 0000000..01fee8a --- /dev/null +++ b/ai-qa/lib/report-footer.test.js @@ -0,0 +1,27 @@ +"use strict"; + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { formatModelFooter, modelFromExecutionLog } = require("./report-footer.js"); + +test("formatModelFooter renders model and re-run hint", () => { + const s = formatModelFooter("claude/claude-sonnet-5"); + assert.match(s, /Model: `claude\/claude-sonnet-5`/); + assert.match(s, /Re-run this job if you need another review pass/); +}); + +test("formatModelFooter returns empty for blank", () => { + assert.equal(formatModelFooter(""), ""); + assert.equal(formatModelFooter(null), ""); +}); + +test("modelFromExecutionLog reads the first named model", () => { + assert.equal( + modelFromExecutionLog([ + { type: "system", subtype: "init", model: "oc/mimo-v2.5-free" }, + { type: "result", subtype: "success" }, + ]), + "oc/mimo-v2.5-free", + ); + assert.equal(modelFromExecutionLog(null), ""); +}); From 87dc3dd8c8062bc7ec260519c1d883b73761313f Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 07:20:39 +0300 Subject: [PATCH 05/10] =?UTF-8?q?docs:=20note=20locked=20Claude=E2=86=92Cu?= =?UTF-8?q?rsor=E2=86=92free=20model=20cascade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #51 --- ai-qa/README.md | 1 - ai-review/README.md | 6 +++++- docs/plan.md | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ai-qa/README.md b/ai-qa/README.md index 6c7034d..9031b5a 100644 --- a/ai-qa/README.md +++ b/ai-qa/README.md @@ -115,7 +115,6 @@ prompt.) | `health-url` | URL polled with `curl --fail` until healthy or `deploy-timeout` elapses; also smoke-tested directly by the review. No sensible generic default exists. | **Yes** | — | | `deploy-timeout` | Seconds to keep polling `health-url` before giving up. | No | `180` | | `test-hint` | Optional free-text describing how to build/test this repo. Handed to the review as context — Claude MAY run it at its discretion to confirm a suspected regression, never mechanically. Consumer must provision the toolchain first. | No | `""` | -| `qa-model` | Model used for the agentic QA review. | No | `claude-sonnet-5` | | `allowed-tools` | Tool allowlist passed to the review's `--allowedTools` (read/grep the code, `curl` the deploy, `git` the diff, optionally run a JS/TS build/test). Override to widen or narrow. | No | *(read/grep/glob + curl/git + node/npm/npx/yarn/pnpm/corepack)* | | `pass-label` | Label applied when the overall QA signal (health + review) passes. Also applied to linked issues when `update-linked-issues` is on. | No | `✓ /ai-qa` | | `fail-label` | Label applied when the overall QA signal fails. Also applied to linked issues (and the merge-auto-closed issue is reopened) when `update-linked-issues` is on. | No | `✗ /ai-qa` | diff --git a/ai-review/README.md b/ai-review/README.md index aea28d4..07b3e42 100644 --- a/ai-review/README.md +++ b/ai-review/README.md @@ -10,7 +10,11 @@ workflow can gate its own heavier build/test/deploy jobs on The verdict comes from a real two-stage AI review: a Haiku context stage summarizes the diff, then a diff-size-routed Sonnet/Opus review stage performs the full rubric scan (see `ai-review/rubric.md`) and returns a -schema-validated structured result. The `Publish review` step never trusts +schema-validated structured result. Model IDs are **locked in the action** +(Claude primary → Cursor if Claude is blocked → pinned OpenCode free models, +then `auto/best-free`). The posted PR review footer names the model that +actually ran and hints to re-run the job for another pass. The `Publish review` +step never trusts the model's self-reported verdict directly — it deterministically recomputes confidence, verdict, and merge risk from the model's reported P0-P3 finding counts and test-quality signals, then posts that as a native diff --git a/docs/plan.md b/docs/plan.md index d2c162c..e215c27 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -29,8 +29,8 @@ Today `ai-review`/`ai-qa` are autopilot **skills** a developer runs in a local C - **Public & generalized:** the repo is **public** and org-agnostic. No action ships an eduly-specific default — eduly values (App ID/key, runner label, health URL, test command, staging URL) are **caller-supplied**; the action's own defaults are neutral (`runner-label: ubuntu-latest`, no health-url/test-command default). READMEs, ADR 0001, and `docs/consumer-integration.md` are written generically with **eduly as one example consumer**, not the subject. A `LICENSE` is a **deferred, blocking decision**: a public repo with no license is "all rights reserved," so external reuse cannot begin until a license is chosen (see D12) — internal EdulyCom consumption is unaffected. - **Process:** all changes via PR (no direct push to protected branches); **merge commits, not squash**. TypeScript/no-`any` for any committed scripts; prefer inline `github-script`. Docs land in the same PR as their change. - **Secrets & variables (org-level, human-supplied):** every secret/var the actions consume is set **at the EdulyCom org level** (`gh secret set --org EdulyCom …` / `gh variable set --org EdulyCom …`, repo-visibility scoped to the consuming repos incl. `github-actions` itself so its `selftest.yml` can run). The **implementer never invents secret values**: at each point one is needed (Phase 1 selftest, Phase 4 eduly adoption) the implementer **pauses and asks the human for the value**, then sets it at org level. Org-level secrets: `ANTHROPIC_AUTH_TOKEN` (eduly gateway bearer), `MTM_BOT_APP_PRIVATE_KEY`. Org-level vars: `ANTHROPIC_BASE_URL` (eduly gateway URL), `MTM_BOT_APP_ID`, `RUNNER_LABEL`, `AI_REVIEW_LABEL` (+ model-ID overrides if a gateway aliases them). (`GITHUB_TOKEN` is auto-provided per repo; not set here.) -- **Model IDs** (repo-var overridable): context `claude-haiku-4-5-20251001`; review defaults to `claude-opus-4-8` (account HAS Opus access), dropping to `claude-sonnet-5` only for **tiny** diffs. Tiny = ≤`sonnet-files-threshold` (default 3) files AND ≤`sonnet-churn-threshold` (default 60) changed lines. (Opus-by-default per ADR 0003; earlier the routing was Sonnet-default for ≤15 files AND ≤400 lines.) -- **Anthropic endpoint — caller-supplied, defaults to Anthropic:** `anthropic-base-url` is an **optional caller input**; when omitted the action leaves `ANTHROPIC_BASE_URL` unset so `claude-code-action` uses the **standard Anthropic endpoint** (no gateway baked in — keeps the action generic per D12). When the caller passes it (eduly → its **custom gateway**), the value is plumbed to **every** `claude-code-action` invocation — Haiku context, Sonnet/Opus review, ai-qa triage — together with the caller's auth: either `anthropic-api-key` (→ `ANTHROPIC_API_KEY`/`x-api-key`) or `anthropic-auth-token` (→ `ANTHROPIC_AUTH_TOKEN` bearer). Because a gateway may expose **aliased** model names, model IDs stay repo-var overridable (a gateway lacking Opus falls back to the Sonnet ID via the override). +- **Model IDs** (locked in the action, not repo-var overridable): context `claude/claude-haiku-4-5-20251001`; review routes tiny diffs to `claude/claude-sonnet-5` and larger diffs to `claude/claude-opus-5`. Cascade on overload: Cursor (`claude/cursor/...`) then pinned OpenCode free models then `auto/best-free`. Tiny = ≤`sonnet-files-threshold` (default 25) files AND ≤`sonnet-churn-threshold` (default 800) changed lines. ai-qa uses locked Sonnet with the same Cursor→free fallback list. See `docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md` / #51. +- **Anthropic endpoint — caller-supplied, defaults to Anthropic:** `anthropic-base-url` is an **optional caller input**; when omitted the action leaves `ANTHROPIC_BASE_URL` unset so `claude-code-action` uses the **standard Anthropic endpoint** (no gateway baked in — keeps the action generic per D12). When the caller passes it (eduly → its **custom gateway**), the value is plumbed to **every** `claude-code-action` invocation — Haiku context, Sonnet/Opus review, ai-qa triage — together with the caller's auth: either `anthropic-api-key` (→ `ANTHROPIC_API_KEY`/`x-api-key`) or `anthropic-auth-token` (→ `ANTHROPIC_AUTH_TOKEN` bearer). Model IDs are locked in the action with a Claude→Cursor→free `--fallback-model` cascade rather than caller overrides. ## Locked decisions From 4d3d83bc438c9e93398b0ca435d95f80848f9fcf Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 07:44:51 +0300 Subject: [PATCH 06/10] fix(ai-qa): derive model footer fallback from locked primary output Refs #51 --- ai-qa/action.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ai-qa/action.yml b/ai-qa/action.yml index a8a0a36..285df08 100644 --- a/ai-qa/action.yml +++ b/ai-qa/action.yml @@ -286,6 +286,21 @@ runs: run: | # zizmor: ignore[github-env] BASE_URL is this action's own anthropic-base-url input (caller-supplied static config), never attacker-influenceable event data — see ADR-0010's env-binding rule echo "ANTHROPIC_BASE_URL=${BASE_URL}" >> "${GITHUB_ENV}" + # Locked model IDs (not inputs). Keep in sync with ai-review cascade. + # Claude primary → Cursor if Claude blocked → free if Cursor blocked. + - name: Resolve locked models + id: models + if: >- + steps.merge.outputs.skip != 'true' && + (inputs.anthropic-api-key != '' || inputs.anthropic-auth-token != '') + shell: bash + run: | + set -euo pipefail + { + echo "primary=claude/claude-sonnet-5" + echo "fallback=claude/cursor/claude-4.6-sonnet-medium-thinking,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free" + } >> "${GITHUB_OUTPUT}" + # Workaround for anthropics/claude-code-action#1294: the action requires # *some* value in `anthropic_api_key` even when authenticating to a custom # gateway via a bearer token, so fall back to the auth token itself and @@ -308,8 +323,8 @@ runs: # The list is ordered and the primary is re-tried at the start of # each turn. Does not bound silent gateway stalls (ADR 0005). claude_args: | - --model claude/claude-sonnet-5 - --fallback-model claude/cursor/claude-4.6-sonnet-medium-thinking,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free + --model ${{ steps.models.outputs.primary }} + --fallback-model ${{ steps.models.outputs.fallback }} --allowedTools "${{ inputs.allowed-tools }}" --json-schema '{"type":"object","additionalProperties":false,"required":["verdict","confidence","merge_risk","deploy_status","counts","summary","report_markdown"],"properties":{"verdict":{"type":"string","enum":["pass","fail"]},"confidence":{"type":"integer","minimum":0,"maximum":100},"merge_risk":{"type":"string","enum":["low","medium","high"]},"deploy_status":{"type":"string","enum":["healthy","unhealthy","unknown"]},"counts":{"type":"object","additionalProperties":false,"required":["p0","p1","p2","p3"],"properties":{"p0":{"type":"integer","minimum":0},"p1":{"type":"integer","minimum":0},"p2":{"type":"integer","minimum":0},"p3":{"type":"integer","minimum":0}}},"summary":{"type":"string"},"report_markdown":{"type":"string"},"test_plan":{"type":"object","additionalProperties":false,"required":["present"],"properties":{"present":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["text","status"],"properties":{"text":{"type":"string"},"status":{"type":"string","enum":["passed","failed","unverifiable"]},"evidence":{"type":"string"}}}}}}}}' prompt: | From 91f2d8b439825a542c9e0711d7d3a912439bb4aa Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 07:46:24 +0300 Subject: [PATCH 07/10] fix(ai-qa): fall back publish Model line to locked primary Prefer the execution-log model when present; otherwise use steps.models.outputs.primary instead of a second Sonnet literal. Refs #51 --- ai-qa/action.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ai-qa/action.yml b/ai-qa/action.yml index 285df08..fdf4c16 100644 --- a/ai-qa/action.yml +++ b/ai-qa/action.yml @@ -401,6 +401,10 @@ runs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} REVIEW_EXEC_FILE: ${{ steps.review.outputs.execution_file }} REPORT_FOOTER_PATH: ${{ github.action_path }}/lib/report-footer.js + # Same locked primary the review stage was started with (steps.models). + # Prefer the execution-log model when present; otherwise the requested + # primary — never a second hardcoded literal that can drift from --model. + QA_PRIMARY_MODEL: ${{ steps.models.outputs.primary }} with: github-token: ${{ steps.identity.outputs.author-token }} script: | @@ -414,8 +418,12 @@ runs: } catch { // No execution log (review skipped / missing). } + // Only attribute a model when we have structured QA output. Prefer the + // log's resolved model (captures Cursor/free fallback). If the log is + // silent, fall back to the locked primary that this run requested — + // same shape as ai-review's resolveModelUsed({ fallback: ROUTED_MODEL }). if (!modelUsed && process.env.QA_OUTPUT) { - modelUsed = 'claude/claude-sonnet-5'; + modelUsed = process.env.QA_PRIMARY_MODEL || ''; } const prNumber = Number(process.env.PR_NUMBER); From 98261acfbc51a9b660fbfad727146e79c701a73c Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 08:17:26 +0300 Subject: [PATCH 08/10] fix(ai-review): use composer-2.5 for Cursor fallback tier claude/cursor/claude-* hangs Claude Code CLI (aborted_streaming) even when /v1/messages succeeds, so a Claude-off cascade never reached free. Cap the chain at three entries to match the CLI limit. Refs #51 --- ai-qa/action.yml | 6 +++-- ai-review/README.md | 4 +-- ai-review/action.yml | 16 +++++++++--- ...-08-17-claude-cursor-free-model-cascade.md | 25 +++++++++++-------- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/ai-qa/action.yml b/ai-qa/action.yml index fdf4c16..de8e7fe 100644 --- a/ai-qa/action.yml +++ b/ai-qa/action.yml @@ -287,7 +287,9 @@ runs: echo "ANTHROPIC_BASE_URL=${BASE_URL}" >> "${GITHUB_ENV}" # Locked model IDs (not inputs). Keep in sync with ai-review cascade. - # Claude primary → Cursor if Claude blocked → free if Cursor blocked. + # Claude primary → Cursor (composer-2.5) if Claude blocked → free. + # Do not use claude/cursor/claude-* here: CLI hangs on those IDs (see + # ai-review Resolve model routing comment). Cap chain at 3 (CLI limit). - name: Resolve locked models id: models if: >- @@ -298,7 +300,7 @@ runs: set -euo pipefail { echo "primary=claude/claude-sonnet-5" - echo "fallback=claude/cursor/claude-4.6-sonnet-medium-thinking,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free" + echo "fallback=claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free" } >> "${GITHUB_OUTPUT}" # Workaround for anthropics/claude-code-action#1294: the action requires diff --git a/ai-review/README.md b/ai-review/README.md index 07b3e42..983f1f1 100644 --- a/ai-review/README.md +++ b/ai-review/README.md @@ -11,8 +11,8 @@ The verdict comes from a real two-stage AI review: a Haiku context stage summarizes the diff, then a diff-size-routed Sonnet/Opus review stage performs the full rubric scan (see `ai-review/rubric.md`) and returns a schema-validated structured result. Model IDs are **locked in the action** -(Claude primary → Cursor if Claude is blocked → pinned OpenCode free models, -then `auto/best-free`). The posted PR review footer names the model that +(Claude primary → Cursor `composer-2.5` if Claude is blocked → OpenCode free / +`auto/best-free`; Claude Code caps the fallback chain at 3). The posted PR review footer names the model that actually ran and hints to re-run the job for another pass. The `Publish review` step never trusts the model's self-reported verdict directly — it deterministically diff --git a/ai-review/action.yml b/ai-review/action.yml index 7afdcf0..f064a07 100644 --- a/ai-review/action.yml +++ b/ai-review/action.yml @@ -421,13 +421,21 @@ runs: mkdir -p .ai-review # Locked model IDs (not inputs). Keep in sync with ai-qa. + # + # Cursor tier MUST be composer-2.5 (or another CLI-proven Cursor ID). + # claude/cursor/claude-* IDs accept raw /v1/messages but hang Claude Code + # CLI (unrecognized_model → aborted_streaming), so a Claude-off cascade + # never reaches free if those IDs are first in --fallback-model. + # Claude Code caps fallback chains at 3 entries after dedupe. SONNET="claude/claude-sonnet-5" OPUS="claude/claude-opus-5" HAIKU="claude/claude-haiku-4-5-20251001" - FREE="oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free" - SONNET_FALLBACK="claude/cursor/claude-4.6-sonnet-medium-thinking,${FREE}" - OPUS_FALLBACK="claude/cursor/claude-opus-4-8-medium-fast,${FREE}" - HAIKU_FALLBACK="claude/cursor/composer-2.5,${FREE}" + CURSOR="claude/cursor/composer-2.5" + FREE="oc/nemotron-3.5-lightning-free,auto/best-free" + FALLBACK_CHAIN="${CURSOR},${FREE}" + SONNET_FALLBACK="${FALLBACK_CHAIN}" + OPUS_FALLBACK="${FALLBACK_CHAIN}" + HAIKU_FALLBACK="${FALLBACK_CHAIN}" export SONNET OPUS HAIKU # Resolve the review range once, deterministically. `git diff A...B` diff --git a/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md b/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md index 7087680..488e062 100644 --- a/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md +++ b/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md @@ -17,7 +17,9 @@ - Keep `sonnet-files-threshold` / `sonnet-churn-threshold` inputs on `ai-review`. - Prefer `claude/` and `claude/cursor/` prefixes (Anthropic Messages–compatible). - Free tail = **pinned usable `oc/*-free` IDs**, then **`auto/best-free` last** (hybrid). Do not use `auto/*` as Claude or Cursor slots. Do not use `auto/coding:free` (unsupported upstream). Prefer `oc/` over duplicate `opencode/` aliases. -- Shared free tail (all roles): `oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free` +- Shared fallback chain (all roles, CLI max 3): `claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free` + - Cursor tier is **composer-2.5 only** — not `claude/cursor/claude-*` (CLI hang) + - Longer free inventories are ignored past the 3-entry CLI cap - `--fallback-model` must not include the primary (CLI no-op otherwise). - Do not claim `--fallback-model` fixes silent ~27m gateway stalls (ADR 0005). - Injection safety: bind values via `env:`, never interpolate attacker-controlled text into `run:`/`script:` bodies with `${{ }}`. @@ -78,16 +80,19 @@ Catalog free-ish IDs (excluding video `veo*`): Shared free tail constant (paste identically everywhere): ```text -FREE=oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free +FREE=oc/nemotron-3.5-lightning-free,auto/best-free +# Full chain (Cursor + FREE) must stay ≤3 entries — Claude Code ignores the rest. ``` Do **not** add `oc/big-pickle` while billing is unknown (not labeled free in the catalog). Revisit only after confirming it is zero-cost on the Eduly gateway. | Action / role | Primary | `--fallback-model` (ordered) | | --- | --- | --- | -| ai-review context | `claude/claude-haiku-4-5-20251001` | `claude/cursor/composer-2.5,` + FREE | -| ai-review review (tiny) | `claude/claude-sonnet-5` | `claude/cursor/claude-4.6-sonnet-medium-thinking,` + FREE | -| ai-review review (large) | `claude/claude-opus-5` | `claude/cursor/claude-opus-4-8-medium-fast,` + FREE | -| ai-qa review | `claude/claude-sonnet-5` | `claude/cursor/claude-4.6-sonnet-medium-thinking,` + FREE | +| ai-review context | `claude/claude-haiku-4-5-20251001` | `claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free` | +| ai-review review (tiny) | `claude/claude-sonnet-5` | same Cursor→free chain (CLI max 3) | +| ai-review review (large) | `claude/claude-opus-5` | same Cursor→free chain (CLI max 3) | +| ai-qa review | `claude/claude-sonnet-5` | same Cursor→free chain (CLI max 3) | + +> **Do not** put `claude/cursor/claude-*` in `--fallback-model`: raw `/v1/messages` may 200, but Claude Code CLI hangs (`unrecognized_model` → `aborted_streaming`) and never reaches free. Use `composer-2.5` as the Cursor tier. Comment footer copy (**successful** reviews / QA comments only): @@ -305,9 +310,9 @@ In **Deterministic prep and model routing**, replace `inputs.*-model` with: SONNET="claude/claude-sonnet-5" OPUS="claude/claude-opus-5" HAIKU="claude/claude-haiku-4-5-20251001" -SONNET_FALLBACK="claude/cursor/claude-4.6-sonnet-medium-thinking,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free" -OPUS_FALLBACK="claude/cursor/claude-opus-4-8-medium-fast,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free" -HAIKU_FALLBACK="claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free" +SONNET_FALLBACK="claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free" +OPUS_FALLBACK="claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free" +HAIKU_FALLBACK="claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free" ``` Export `HAIKU` / `OPUS` / `SONNET` to the environment for `write-manifest.js` (same as today). After choosing `MODEL`, also set `FALLBACK` to the matching list and write: @@ -432,7 +437,7 @@ In `ai-qa/action.yml`, delete `qa-model`. Replace review `claude_args` model lin # Locked at action level — keep in sync with ai-review cascade. # Claude primary → Cursor if Claude blocked → free if Cursor blocked. --model claude/claude-sonnet-5 ---fallback-model claude/cursor/claude-4.6-sonnet-medium-thinking,oc/nemotron-3.5-lightning-free,oc/nemotron-3-ultra-free,oc/deepseek-v4-flash-free,oc/mimo-v2.5-free,oc/laguna-s-2.1-free,auto/best-free +--fallback-model claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free ``` Update the comment that currently describes `claude-sonnet-4-6,claude-opus-4-8`. From 447846e0b47ff1214c704d72415e3f7fd6df7d67 Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 08:45:08 +0300 Subject: [PATCH 09/10] fix(ai-review): recover session_id for SO repair; schema free fallbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repair was skipped when claude-code-action failed on missing structured_output without exporting session_id. Recover it from the execution snapshot. Schema stages fall back to structured_output free models only — Cursor has no SO and can retract schema mid-stream. Refs #51 --- ai-qa/action.yml | 9 ++- ai-review/README.md | 5 +- ai-review/action.yml | 66 +++++++++++++++---- ...-08-17-claude-cursor-free-model-cascade.md | 11 ++-- 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/ai-qa/action.yml b/ai-qa/action.yml index de8e7fe..c7dfaab 100644 --- a/ai-qa/action.yml +++ b/ai-qa/action.yml @@ -286,10 +286,9 @@ runs: run: | # zizmor: ignore[github-env] BASE_URL is this action's own anthropic-base-url input (caller-supplied static config), never attacker-influenceable event data — see ADR-0010's env-binding rule echo "ANTHROPIC_BASE_URL=${BASE_URL}" >> "${GITHUB_ENV}" - # Locked model IDs (not inputs). Keep in sync with ai-review cascade. - # Claude primary → Cursor (composer-2.5) if Claude blocked → free. - # Do not use claude/cursor/claude-* here: CLI hangs on those IDs (see - # ai-review Resolve model routing comment). Cap chain at 3 (CLI limit). + # Locked model IDs (not inputs). Keep in sync with ai-review's *schema* + # cascade: Claude → structured_output free only (no Cursor — none advertise + # structured_output, and --json-schema reviews go inconclusive on them). - name: Resolve locked models id: models if: >- @@ -300,7 +299,7 @@ runs: set -euo pipefail { echo "primary=claude/claude-sonnet-5" - echo "fallback=claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free" + echo "fallback=oc/nemotron-3.5-lightning-free,oc/deepseek-v4-flash-free,auto/best-free" } >> "${GITHUB_OUTPUT}" # Workaround for anthropics/claude-code-action#1294: the action requires diff --git a/ai-review/README.md b/ai-review/README.md index 983f1f1..574563d 100644 --- a/ai-review/README.md +++ b/ai-review/README.md @@ -11,8 +11,9 @@ The verdict comes from a real two-stage AI review: a Haiku context stage summarizes the diff, then a diff-size-routed Sonnet/Opus review stage performs the full rubric scan (see `ai-review/rubric.md`) and returns a schema-validated structured result. Model IDs are **locked in the action** -(Claude primary → Cursor `composer-2.5` if Claude is blocked → OpenCode free / -`auto/best-free`; Claude Code caps the fallback chain at 3). The posted PR review footer names the model that +(Claude primary → for context, Cursor `composer-2.5` then free; for schema +reviews, structured_output free models only — Cursor has no SO on this +gateway). The posted PR review footer names the model that actually ran and hints to re-run the job for another pass. The `Publish review` step never trusts the model's self-reported verdict directly — it deterministically diff --git a/ai-review/action.yml b/ai-review/action.yml index f064a07..78f7c35 100644 --- a/ai-review/action.yml +++ b/ai-review/action.yml @@ -422,20 +422,22 @@ runs: # Locked model IDs (not inputs). Keep in sync with ai-qa. # - # Cursor tier MUST be composer-2.5 (or another CLI-proven Cursor ID). - # claude/cursor/claude-* IDs accept raw /v1/messages but hang Claude Code - # CLI (unrecognized_model → aborted_streaming), so a Claude-off cascade - # never reaches free if those IDs are first in --fallback-model. - # Claude Code caps fallback chains at 3 entries after dedupe. + # Two cascades (Claude Code caps --fallback-model at 3 entries): + # Context (no --json-schema): Claude → composer-2.5 → free. + # composer is CLI-safe; claude/cursor/claude-* hang the CLI. + # Review (has --json-schema): Claude → structured_output free only. + # No Cursor model advertises structured_output on this gateway, and + # Agent SDK docs: a mid-stream model fallback can retract an already + # completed structured_output — landing on composer then yields the + # inconclusive "success but no structured_output" failure mode. SONNET="claude/claude-sonnet-5" OPUS="claude/claude-opus-5" HAIKU="claude/claude-haiku-4-5-20251001" CURSOR="claude/cursor/composer-2.5" - FREE="oc/nemotron-3.5-lightning-free,auto/best-free" - FALLBACK_CHAIN="${CURSOR},${FREE}" - SONNET_FALLBACK="${FALLBACK_CHAIN}" - OPUS_FALLBACK="${FALLBACK_CHAIN}" - HAIKU_FALLBACK="${FALLBACK_CHAIN}" + FREE_SO="oc/nemotron-3.5-lightning-free,oc/deepseek-v4-flash-free,auto/best-free" + HAIKU_FALLBACK="${CURSOR},oc/nemotron-3.5-lightning-free,auto/best-free" + SONNET_FALLBACK="${FREE_SO}" + OPUS_FALLBACK="${FREE_SO}" export SONNET OPUS HAIKU # Resolve the review range once, deterministically. `git diff A...B` @@ -870,11 +872,18 @@ runs: if: always() && steps.fork-guard.outputs.is-fork != 'true' && steps.pr-state.outputs.skip != 'true' shell: bash env: + # Prefer the action output; fall back to the fixed path the pinned + # claude-code-action always writes (outputs can be missing when the + # step fails on a missing structured_output after logging session_id). EXEC_FILE: ${{ steps.review.outputs.execution_file }} run: | set -uo pipefail - if [ -n "${EXEC_FILE:-}" ] && [ -s "${EXEC_FILE}" ]; then - cp "${EXEC_FILE}" "${RUNNER_TEMP}/ai-review-exec-review-snapshot.json" || true + CANDIDATE="${EXEC_FILE:-}" + if [ -z "${CANDIDATE}" ] || [ ! -s "${CANDIDATE}" ]; then + CANDIDATE="${RUNNER_TEMP}/claude-execution-output.json" + fi + if [ -s "${CANDIDATE}" ]; then + cp "${CANDIDATE}" "${RUNNER_TEMP}/ai-review-exec-review-snapshot.json" || true fi exit 0 @@ -890,6 +899,11 @@ runs: # wins, and its duration_ms is the stage duration. Best-effort — any # parse failure leaves retry-ok=true, so this can only ever suppress a # retry that is provably too late, never block one on missing data. + # + # Also recovers session_id for the cheap repair step: when the review + # action fails on missing structured_output it may log "Set session_id" + # without exporting the output, which previously skipped repair entirely + # (observed on run 31997386554) and forced a full re-review. - name: Gate the review retry on elapsed review time id: retry_budget if: always() && steps.fork-guard.outputs.is-fork != 'true' && steps.pr-state.outputs.skip != 'true' @@ -898,14 +912,34 @@ runs: # 15 min. A retry starting later than this cannot complete within the # 25-min timeout docs/consumer-integration.md recommends. RETRY_BUDGET_MS: "900000" + REVIEW_SESSION_ID: ${{ steps.review.outputs.session_id }} run: | set -uo pipefail SNAPSHOT="${RUNNER_TEMP}/ai-review-exec-review-snapshot.json" RETRY_OK=true DURATION_MS=0 + SESSION_ID="${REVIEW_SESSION_ID:-}" if [ -s "${SNAPSHOT}" ]; then DURATION_MS="$(jq -r '[.[] | select(.type == "result")] | last | .duration_ms // 0' \ "${SNAPSHOT}" 2>/dev/null || echo 0)" + if [ -z "${SESSION_ID}" ]; then + SESSION_ID="$(jq -r ' + [ + .[] + | select(.type == "system" and .subtype == "init") + | .session_id // empty + ] + | map(select(length > 0)) + | last // empty + ' "${SNAPSHOT}" 2>/dev/null || true)" + fi + if [ -z "${SESSION_ID}" ]; then + SESSION_ID="$(jq -r ' + [.[] | select(.type == "result") | .session_id // empty] + | map(select(length > 0)) + | last // empty + ' "${SNAPSHOT}" 2>/dev/null || true)" + fi fi case "${DURATION_MS}" in ''|*[!0-9]*) DURATION_MS=0 ;; @@ -914,9 +948,13 @@ runs: RETRY_OK=false echo "::warning::Review stage ran ${DURATION_MS}ms (>= ${RETRY_BUDGET_MS}ms budget); skipping the full-review retry, which could not finish inside the caller's timeout." fi + if [ -z "${SESSION_ID}" ]; then + echo "::warning::No review session_id in step outputs or execution snapshot; structured-output repair will be skipped." + fi { echo "retry-ok=${RETRY_OK}" echo "review-duration-ms=${DURATION_MS}" + echo "session_id=${SESSION_ID}" } >> "${GITHUB_OUTPUT}" exit 0 @@ -935,7 +973,7 @@ runs: steps.fork-guard.outputs.is-fork != 'true' && steps.pr-state.outputs.skip != 'true' && steps.review.outputs.structured_output == '' && - steps.review.outputs.session_id != '' + steps.retry_budget.outputs.session_id != '' uses: anthropics/claude-code-action@6b082c41935b4c8a3b8b0ef85ba4ba4d9eeb8975 # v1.0.189 env: ANTHROPIC_BASE_URL: ${{ inputs.anthropic-base-url }} @@ -946,7 +984,7 @@ runs: github_token: ${{ inputs.github-token }} allowed_bots: ${{ inputs.allowed-bots }} claude_args: | - --resume ${{ steps.review.outputs.session_id }} + --resume ${{ steps.retry_budget.outputs.session_id }} --model ${{ steps.route.outputs.model }} --fallback-model ${{ steps.route.outputs.fallback-model }} --allowedTools "Read" diff --git a/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md b/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md index 488e062..9edaa87 100644 --- a/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md +++ b/docs/superpowers/plans/2026-08-17-claude-cursor-free-model-cascade.md @@ -88,11 +88,14 @@ Do **not** add `oc/big-pickle` while billing is unknown (not labeled free in the | Action / role | Primary | `--fallback-model` (ordered) | | --- | --- | --- | | ai-review context | `claude/claude-haiku-4-5-20251001` | `claude/cursor/composer-2.5,oc/nemotron-3.5-lightning-free,auto/best-free` | -| ai-review review (tiny) | `claude/claude-sonnet-5` | same Cursor→free chain (CLI max 3) | -| ai-review review (large) | `claude/claude-opus-5` | same Cursor→free chain (CLI max 3) | -| ai-qa review | `claude/claude-sonnet-5` | same Cursor→free chain (CLI max 3) | +| ai-review review (tiny) | `claude/claude-sonnet-5` | `oc/nemotron-3.5-lightning-free,oc/deepseek-v4-flash-free,auto/best-free` (SO free only) | +| ai-review review (large) | `claude/claude-opus-5` | same SO free chain | +| ai-qa review | `claude/claude-sonnet-5` | same SO free chain | + +> **Do not** put `claude/cursor/claude-*` in `--fallback-model`: raw `/v1/messages` may 200, but Claude Code CLI hangs (`unrecognized_model` → `aborted_streaming`). +> **Do not** put Cursor (including `composer-2.5`) on `--json-schema` review/QA stages: no Cursor ID advertises `structured_output` on this gateway, and Agent SDK docs note a mid-stream fallback can retract structured output — that is the inconclusive path. +> Context (no schema) may still use `composer-2.5` as the Cursor tier. -> **Do not** put `claude/cursor/claude-*` in `--fallback-model`: raw `/v1/messages` may 200, but Claude Code CLI hangs (`unrecognized_model` → `aborted_streaming`) and never reaches free. Use `composer-2.5` as the Cursor tier. Comment footer copy (**successful** reviews / QA comments only): From dd8f5ab2da0a753b444dc98b875d7544d5e4c021 Mon Sep 17 00:00:00 2001 From: Hussam Aldarwish Date: Mon, 17 Aug 2026 12:08:38 +0300 Subject: [PATCH 10/10] fix(ai-review): unescape literal \\n in comment_markdown before publish Some structured outputs deliver comment_markdown with two-character "\n" sequences (seen on PR #52 review 4949356509), which GitHub renders as one smashed block under the banner. Refs #51 --- ai-review/lib/publish.js | 17 +++++++++++++++++ ai-review/lib/publish.test.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/ai-review/lib/publish.js b/ai-review/lib/publish.js index a9f1f89..b04a5d6 100644 --- a/ai-review/lib/publish.js +++ b/ai-review/lib/publish.js @@ -13,6 +13,20 @@ const STATUS_BLOCK_START = ""; const STATUS_BLOCK_END = ""; +// Some structured-output paths deliver comment_markdown with literal +// two-character "\n" sequences instead of real newlines (observed on +// EdulyCom/github-actions#52 review 4949356509). GitHub then renders the +// findings as one smashed line under the banner. Only rewrite when the +// literal escapes dominate real newlines so a normal body that mentions +// `\n` in prose/code is left alone. +function unescapeLiteralNewlines(markdown) { + if (typeof markdown !== "string" || !markdown.includes("\\n")) return markdown; + const literal = (markdown.match(/\\n/g) || []).length; + const real = (markdown.match(/\n/g) || []).length; + if (literal === 0 || literal < real) return markdown; + return markdown.replace(/\\r\\n/g, "\n").replace(/\\n/g, "\n").replace(/\\t/g, "\t"); +} + // The review-stage prompt instructs the model not to prepend its own // verdict token / confidence-merge-risk line / HTML marker to // comment_markdown (the caller owns that banner), but model @@ -21,6 +35,7 @@ const STATUS_BLOCK_END = ""; // duplicate the banner above it. function stripLeadingBannerArtifacts(markdown) { if (!markdown) return markdown; + markdown = unescapeLiteralNewlines(markdown); const verdictTokenRe = /^\*\*(?:✅ PASS|❌ FAIL)\*\*\s*$/; const confidenceLineRe = /^Confidence:\s*\d+\s*·\s*Merge risk:\s*\S+\s*$/i; const htmlCommentRe = /^\s*$/; @@ -116,6 +131,7 @@ function buildReviewBody({ */ function buildInconclusiveBody(salvaged, opts = {}) { const modelUsed = opts && opts.modelUsed; + salvaged = unescapeLiteralNewlines(salvaged || ""); return [ "", "### ⚠️ AI Review — inconclusive (re-run required)", @@ -224,6 +240,7 @@ function upsertStatusBlock(body, block) { } module.exports = { + unescapeLiteralNewlines, stripLeadingBannerArtifacts, buildReviewBody, buildInconclusiveBody, diff --git a/ai-review/lib/publish.test.js b/ai-review/lib/publish.test.js index eea4f8f..3fcd932 100644 --- a/ai-review/lib/publish.test.js +++ b/ai-review/lib/publish.test.js @@ -14,6 +14,37 @@ const { // --- stripLeadingBannerArtifacts -------------------------------------------- +test("unescapes literal \\\\n when comment_markdown was double-escaped", () => { + // Observed on PR #52 review 4949356509: model/schema path left `\\n` as + // two characters, so GitHub rendered the body as one smashed line. + const raw = + "### P0 — Blockers\\n\\n_None._\\n\\n### P1 — Should Fix\\n\\n_None._\\n\\n### P2 — Nice-to-Have\\n\\n- drift risk"; + const out = stripLeadingBannerArtifacts(raw); + assert.equal( + out, + [ + "### P0 — Blockers", + "", + "_None._", + "", + "### P1 — Should Fix", + "", + "_None._", + "", + "### P2 — Nice-to-Have", + "", + "- drift risk", + ].join("\n") + ); + assert.equal(out.includes("\\n"), false); +}); + +test("leaves normal markdown with real newlines alone", () => { + const raw = "### P0 — Blockers\n\n_None._\n\nUse `\\\\n` in a code span occasionally."; + const out = stripLeadingBannerArtifacts(raw); + assert.equal(out, raw); +}); + test("strips a leading verdict token line", () => { const out = stripLeadingBannerArtifacts("**✅ PASS**\n\nReal content here."); assert.equal(out, "Real content here.");