1.4

.gitignore

@@ -38,7 +38,5 @@ Thumbs.db
 .env
 .env.local
 
-# CommitBrief runtime artifacts when dogfooding the tool inside its own repo
-# (.commitbrief/config.yml is per-user; cache/ is local-only)
-/.commitbrief/
-reviews/
+# CommitBrief local config and cache
+.commitbrief/

CHANGELOG.md

@@ -8,6 +8,91 @@ and the project adheres to [Semantic Versioning 2.0.0](https://semver.org/spec/v
 > Tags prior to **v0.4.0** were cut in the private repository and produced no
 > public artifacts; the first publicly released version is v0.4.0.
 
+## [1.4.0] - 2026-05-29
+
+### Fixed
+- **Progress spinner floods the screen (repeats a stage line every frame).**
+  The animated renderer redraws in place by moving the cursor up
+  `prevLen` *logical* lines, but a stage line longer than the terminal
+  width wraps to multiple *physical* rows — so the cursor-up under-counted,
+  the tree marched downward, and the top line (e.g. "Searching for
+  changes…") was left behind on every frame. The long `--with-context`
+  security-warning line triggered this on normal-width terminals. Rendered
+  lines are now clipped to the terminal width so they never wrap, keeping
+  the in-place redraw exact. Additionally, `TERM=dumb` terminals (emacs
+  `M-x shell`, some IDE consoles) — which report as a TTY but ignore
+  cursor-movement escapes — now fall back to plain mode. Workaround on
+  older builds: `--color never` or `NO_COLOR=1`.
+- **`isRetriable` (eval harness) matched HTTP codes as bare substrings.** A
+  non-transient error embedding `500`/`503` in a token count or duration
+  (e.g. "requested 130500 tokens", "1500ms") was wrongly retried as a
+  billable live call. Status codes now match on a digit boundary.
+
+### Added
+- **Live elapsed-time counter on the active progress stage.** Once a stage
+  has run for more than a second, the animated tree shows a muted timer
+  beside it (e.g. `Thinking… 0:42`), so a slow `--with-context` agent call
+  reads as working rather than frozen. Fast stages stay clean; the timer's
+  width is reserved out of the line budget so it never causes wrapping.
+- **`--show-prompt` flag.** Prints the exact system + user prompt that
+  would be sent to the model, then exits — no provider call, no cache
+  lookup, no cost. Reflects every prompt-shaping flag (scope,
+  `--with-context`, `--cli`/`--provider`, `--lang`) and honours `--output`.
+  A transparency inspector for "what exactly leaves my machine?" (v1.4).
+- **`guard.token_preflight` config (opt-in, default false).** When on, a
+  review whose estimated prompt tokens exceed the provider's context
+  window prompts for confirmation (TTY) or aborts (non-TTY) before the
+  paid round-trip, instead of letting the provider reject it with a raw
+  400. Off by default — the estimate is a chars/4 heuristic (ADR-0003, v1.4).
+- **Review-quality eval harness (`make eval`).** A maintainer-facing
+  harness that scores actual review output against a curated known-answer
+  corpus and reports precision / recall / false-positive rate (ADR-0018,
+  v1.4.0 "Trust & quality"). The corpus lives at
+  `internal/eval/testdata/corpus/<name>/` — one directory per fixture with
+  `input.diff` (the change under review), `expected.json` (the answer key),
+  and `mock_response.json` (scripted findings for the deterministic tier).
+  Scoring matches produced findings on file + line-tolerance + severity
+  floor and reuses the locked `--json` schema v1 `findings[]` (no new
+  output contract). Two tiers: `make eval` runs the mock provider over the
+  corpus (deterministic, runs in plain `go test ./...`, validates the
+  harness + matcher — part of CI) and `make eval-live` runs a real provider
+  (resolved from `COMMITBRIEF_EVAL_PROVIDER`/`COMMITBRIEF_EVAL_API_KEY` or,
+  with no env vars, the default provider in `~/.commitbrief/config.yml`;
+  behind the `live` build tag, non-deterministic, the source of README
+  quality numbers — never a CI gate). Ships with a 23-fixture seed corpus
+  spanning security (SQL/command/path/SSRF/XSS injection, weak crypto,
+  hardcoded secret), correctness (nil deref, off-by-one, unchecked type
+  assert, mutable default arg), concurrency (data race, WaitGroup misuse),
+  resource leaks (unclosed file/response-body/SQL-rows), error handling
+  (swallowed error, bare except, panic-on-input), a performance case, and
+  three clean controls (rename, comment-typo, added test) that must stay
+  silent. Several fixtures annotate more than one expected finding where the
+  diff genuinely contains secondary defects (e.g. a second panic, an ignored
+  `rows.Scan` error, a truncating fixed-buffer read).
+- **`make eval-dump` diagnostic.** Prints every finding a live provider
+  produces per fixture, tagged `match` / `EXTRA`, to decide whether an
+  EXTRA is a legitimate secondary defect to annotate or genuine noise to
+  leave as a measured false positive.
+- **Held-out slice (Goodhart protection).** A fixture can set
+  `"held_out": true`; ~26% of the corpus (6/23, spanning all defect
+  categories + a clean control) is held out from any prompt/corpus tuning.
+  `make eval-live` reports FULL / DEV / HELD-OUT scorecards separately, and
+  the deterministic `TestHeldOutSlice` fails if the slice is emptied,
+  drops below 15%, or stops being representative — so the protection
+  cannot be silently disabled (ADR-0018 §Goodhart).
+- **`COMMITBRIEF_EVAL_PROVIDER` / `COMMITBRIEF_EVAL_MODEL` overrides.**
+  Select the eval provider and model via env while the API key is read
+  from `~/.commitbrief/config.yml`, so one config benchmarks every
+  provider/model without putting a key on the command line. `RunCorpus`
+  retries each fixture (linear backoff) to ride over transient provider
+  503s during a run.
+- **README "Measured review quality" table.** First published scorecard
+  across five models (Haiku 4.5 / Sonnet 4.6 / Opus 4.8 / Gemini 2.5 Flash
+  / GPT-4o, 2026-05-29), each cell reported as `dev · held` (tunable slice
+  vs held-out generalization slice). Every model recalls the full held-out
+  slice; precision 0.48–0.84 is a conservative floor (recall + FP-rate are
+  the cleaner signals).
+
 ## [1.3.0]
 
 ### Added

Makefile

@@ -13,7 +13,7 @@ LDFLAGS := -s -w \
 
 GO ?= go
 
-.PHONY: help build test test-live bench lint fmt tidy clean check release-check license-check i18n-check spdx-check security-check manpage smoke
+.PHONY: help build test test-live eval eval-live bench lint fmt tidy clean check release-check license-check i18n-check spdx-check security-check manpage smoke
 
 help: ## Show this help
 	@awk 'BEGIN {FS = ":.*## "} /^[a-zA-Z_-]+:.*## / {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -27,6 +27,15 @@ test: ## Run unit + integration tests (live provider tests excluded)
 test-live: ## Run live provider tests (real API keys required)
 	$(GO) test -tags=live ./...
 
+eval: ## Deterministic mock-tier review-quality eval (CI-safe; ADR-0018)
+	$(GO) test ./internal/eval/ -run TestEvalMockCorpus -v
+
+eval-live: ## Live-provider review-quality eval (uses COMMITBRIEF_EVAL_PROVIDER or ~/.commitbrief/config.yml)
+	$(GO) test -tags=live -count=1 -timeout=20m ./internal/eval/ -run '^TestEvalLive$$' -v
+
+eval-dump: ## Diagnostic: print every finding a live provider produces per fixture (match/EXTRA)
+	$(GO) test -tags=live -count=1 -timeout=20m ./internal/eval/ -run '^TestEvalLiveDump$$' -v
+
 bench: ## Run local-pipeline + cache benchmarks (PRD §7.1 targets)
 	$(GO) test -bench=. -benchmem -run=^$$ ./internal/diff ./internal/cache
 

README.md

@@ -37,6 +37,46 @@ read on your diff before another human (or your future self) sees it.
   system prompt; per-user `OUTPUT.md` controls how findings are
   formatted.
 
+## Measured review quality
+
+CommitBrief ships an eval harness (`make eval`) that scores real review
+output against a 23-fixture known-answer corpus — 23 planted defects
+across security, correctness, concurrency, resource-leak, error-handling
+and performance categories, plus 3 clean controls a good review must stay
+silent on. About a quarter of the corpus is a **held-out slice** that
+prompt and corpus tuning never inspect, so each cell below reports
+`dev · held` — the tunable slice and the held-out generalization slice
+separately (ADR-0018). Numbers are from `make eval-live`, captured
+2026-05-29 (mean of *Runs* live runs each):
+
+| Model              | Recall (dev · held) | FP-rate (dev · held) | Precision (dev · held) | Runs |
+|--------------------|:-------------------:|:--------------------:|:----------------------:|:----:|
+| Claude Haiku 4.5   | 1.00 · 1.00         | 0.00 · 0.00          | 0.70 · 0.62            | 5    |
+| Claude Sonnet 4.6  | 1.00 · 1.00         | 0.00 · 0.50          | 0.68 · 0.48            | 3    |
+| Claude Opus 4.8    | 0.94 · 1.00         | 0.00 · 0.00          | 0.61 · 0.53            | 3    |
+| Gemini 2.5 Flash   | 0.96 · 1.00         | 0.44 · 0.00          | 0.84 · 0.56            | 3    |
+| OpenAI GPT-4o      | 0.85 · 1.00         | 0.44 · 0.33          | 0.79 · 0.75            | 3    |
+
+- **Recall** — share of planted defects caught. Every model recalls the
+  full held-out slice; the dev dips (Opus, GPT-4o) come from the harder
+  multi-finding dev fixtures, not from missing whole defects.
+- **FP-rate** — findings landing on a clean-control line (flagging a benign
+  change). Note where the noise lives: Sonnet trips the held-out clean
+  control; Gemini and GPT-4o trip the dev ones.
+- **Precision** — a *conservative floor*: any finding outside the answer
+  key counts as a false positive, but on these small diffs many "extra"
+  findings are legitimate secondary observations (a second panic, an
+  ignored error) rather than noise. The terser models (GPT-4o, Gemini)
+  score higher precisely because they say less — at the cost of recall.
+  Read recall + FP-rate as the cleaner signals; precision is sensitive to
+  how exhaustively the corpus is annotated.
+
+The two slices are **not difficulty-matched** — the split exists to catch
+overfitting in *future* tuning (a dev gain that doesn't carry to held-out),
+not for a direct dev-vs-held comparison today. Reproduce any row with
+`COMMITBRIEF_EVAL_PROVIDER=<name> make eval-live`, which prints FULL / DEV /
+HELD-OUT scorecards (using the key already in `~/.commitbrief/config.yml`).
+
 ## Install
 
 ### Homebrew (macOS / Linux)
@@ -191,7 +231,9 @@ CLI-tool-backed providers; mutually exclusive with `--json` /
 `--markdown`), `--with-context` (CLI providers only — let the host CLI
 read project files beyond the diff to ground the review; see below),
 `--allow-secrets` (acknowledge a flagged credential in
-the diff), `--no-cost-check` (skip cost preflight), `--color`. See
+the diff), `--no-cost-check` (skip cost preflight),
+`--show-prompt` (print the exact system + user prompt that would be sent,
+then exit — no provider call, no cost; honours `--output`), `--color`. See
 `commitbrief --help`.
 
 ### `--with-context` (CLI providers only)
@@ -355,6 +397,9 @@ cache:
   enabled: true
   ttl_days: 7
   max_size_mb: 0                   # 0 = unlimited; >0 evicts oldest entries past the cap
+guard:
+  secret_scan: true                # scan diff + rules for credential patterns before sending
+  token_preflight: false           # opt-in: confirm/abort when the prompt overflows the model's context window
 command:
   default: ""                      # args applied to a bare `commitbrief`; empty = `--staged`
 ```

internal/cli/config.go

@@ -196,8 +196,10 @@ func configFieldGet(cfg *config.Config, path string) (string, error) {
 		switch parts[1] {
 		case "secret_scan":
 			return strconv.FormatBool(cfg.Guard.SecretScan), nil
+		case "token_preflight":
+			return strconv.FormatBool(cfg.Guard.TokenPreflight), nil
 		default:
-			return "", fmt.Errorf("config: unknown field %q in guard (allowed: secret_scan)", parts[1])
+			return "", fmt.Errorf("config: unknown field %q in guard (allowed: secret_scan, token_preflight)", parts[1])
 		}
 
 	case "cost":
@@ -339,8 +341,14 @@ func configFieldSet(cfg *config.Config, path, value string) error {
 				return fmt.Errorf("config: guard.secret_scan: %w", err)
 			}
 			cfg.Guard.SecretScan = b
+		case "token_preflight":
+			b, err := parseConfigBool(value)
+			if err != nil {
+				return fmt.Errorf("config: guard.token_preflight: %w", err)
+			}
+			cfg.Guard.TokenPreflight = b
 		default:
-			return fmt.Errorf("config: unknown field %q in guard (allowed: secret_scan)", parts[1])
+			return fmt.Errorf("config: unknown field %q in guard (allowed: secret_scan, token_preflight)", parts[1])
 		}
 		return nil
 

internal/cli/prompt_preflight_test.go

@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package cli
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/CommitBrief/commitbrief/internal/config"
+	"github.com/CommitBrief/commitbrief/internal/prompt"
+	"github.com/CommitBrief/commitbrief/internal/provider/mock"
+)
+
+// ---------- token preflight (guard.token_preflight, ADR-0003) ----------
+
+func TestHandleTokenPreflightWithinWindowSilent(t *testing.T) {
+	resetGlobalFlags(t)
+	cmd, errBuf := stubCmd(t)
+	app := stubApp(t, 0)
+
+	prov := mock.New() // default ContextWindow is 100_000
+	p := prompt.Prompt{System: "short system", User: "short user"}
+
+	if handleTokenPreflight(cmd, app, prov, p, "mock-model", emptyStdin()) {
+		t.Error("prompt that fits the window must not abort")
+	}
+	if errBuf.Len() > 0 {
+		t.Errorf("within-window preflight must be silent; got stderr:\n%s", errBuf.String())
+	}
+}
+
+func TestHandleTokenPreflightExceedsNonInteractiveAborts(t *testing.T) {
+	resetGlobalFlags(t)
+	cmd, errBuf := stubCmd(t)
+	app := stubApp(t, 0)
+
+	prov := mock.New()
+	prov.Window = 10 // tiny context window
+
+	// EstimatedTokens is chars/4; a few hundred chars easily clears 10.
+	p := prompt.Prompt{
+		System: strings.Repeat("system prompt content ", 40),
+		User:   strings.Repeat("diff line content ", 40),
+	}
+	if !p.ExceedsContext(prov.ContextWindow("mock-model")) {
+		t.Fatal("test setup: prompt should exceed the tiny window")
+	}
+
+	// Test stdin is not a TTY → non-interactive abort path.
+	if !handleTokenPreflight(cmd, app, prov, p, "mock-model", emptyStdin()) {
+		t.Error("over-window prompt in non-interactive mode must abort")
+	}
+	got := errBuf.String()
+	if !strings.Contains(got, "context window") {
+		t.Errorf("expected the over-window warning on stderr; got:\n%s", got)
+	}
+	if !strings.Contains(got, "non-interactive") {
+		t.Errorf("expected the non-interactive abort notice; got:\n%s", got)
+	}
+}
+
+// ---------- guard.token_preflight config round-trip ----------
+
+func TestConfigFieldTokenPreflightRoundTrip(t *testing.T) {
+	cfg := config.Default()
+
+	// Default is opt-in → false.
+	if got, err := configFieldGet(cfg, "guard.token_preflight"); err != nil || got != "false" {
+		t.Fatalf("default guard.token_preflight = %q, err=%v; want \"false\"", got, err)
+	}
+
+	if err := configFieldSet(cfg, "guard.token_preflight", "true"); err != nil {
+		t.Fatalf("set guard.token_preflight: %v", err)
+	}
+	if !cfg.Guard.TokenPreflight {
+		t.Error("set did not flip the struct field")
+	}
+	if got, err := configFieldGet(cfg, "guard.token_preflight"); err != nil || got != "true" {
+		t.Errorf("after set, guard.token_preflight = %q, err=%v; want \"true\"", got, err)
+	}
+}
+
+func TestConfigFieldGuardUnknownFieldListsTokenPreflight(t *testing.T) {
+	cfg := config.Default()
+	_, err := configFieldGet(cfg, "guard.bogus")
+	if err == nil || !strings.Contains(err.Error(), "token_preflight") {
+		t.Errorf("unknown guard field error should list token_preflight; got: %v", err)
+	}
+}
+
+// ---------- --show-prompt ----------
+
+func TestShowPromptEmitsPromptAndSkipsProvider(t *testing.T) {
+	e := newCLIEnv(t)
+
+	if err := e.run("--staged", "--show-prompt"); err != nil {
+		t.Fatalf("--show-prompt: %v\nstderr:\n%s", err, e.errOut.String())
+	}
+	out := e.out.String()
+
+	for _, want := range []string{"===== SYSTEM PROMPT =====", "===== USER PROMPT ====="} {
+		if !strings.Contains(out, want) {
+			t.Errorf("--show-prompt output missing %q; got:\n%s", want, out)
+		}
+	}
+	// The staged diff body must appear in the user prompt.
+	if !strings.Contains(out, "func Login") {
+		t.Errorf("--show-prompt should include the staged diff; got:\n%s", out)
+	}
+	// Proof no review ran: the mock provider's canned finding title must
+	// not appear — --show-prompt exits before any provider call.
+	if strings.Contains(out, "mock review output") {
+		t.Errorf("--show-prompt must not invoke the provider; saw mock review output:\n%s", out)
+	}
+}