feat(flaky): deterministic flaky-test / AI-test anti-pattern pre-pass (ADR-0022)

CHANGELOG.md

@@ -10,6 +10,20 @@ and the project adheres to [Semantic Versioning 2.0.0](https://semver.org/spec/v
 
 ## [Unreleased]
 
+### Added
+- **Deterministic flaky-test detector (ADR-0022).** A static, provider-free
+  pre-pass scans the added lines of changed test files for high-precision
+  flakiness anti-patterns — hard-coded sleeps / fixed waits (`time.Sleep`,
+  `Thread.sleep`, `Task.Delay`, `asyncio.sleep`, `*.waitForTimeout`, numeric
+  `cy.wait`, `usleep`, `sleep(<n>)`) and unseeded randomness (`Math.random`,
+  Python `random.*`, Go `math/rand`) — and merges them into the structured
+  findings, so they render, count toward `--fail-on`, and `--copy` like any
+  other finding. Deterministic and reproducible: no model call, no JSON-schema
+  change (the findings contract stays v1). On by default for API/mock
+  providers; skip per-run with `--no-flaky` or persistently with
+  `review.flaky: false`. Localized (en/tr). CLI-tool-backed plain-text
+  providers are unaffected for now.
+
 ## [1.6.0] - 2026-06-13
 
 ### Added

README.md

@@ -244,8 +244,22 @@ read project files beyond the diff to ground the review; see below),
 `--allow-secrets` (acknowledge a flagged credential in
 the diff), `--no-cost-check` (skip cost preflight),
 `--show-prompt` (print the exact system + user prompt that would be sent,
-then exit — no provider call, no cost; honours `--output`), `--color`. See
-`commitbrief --help`.
+then exit — no provider call, no cost; honours `--output`), `--no-flaky`
+(skip the flaky-test detector below), `--color`. See `commitbrief --help`.
+
+### Flaky-test detection (deterministic, ADR-0022)
+
+Before the model is called, a **static pre-pass** scans the added lines of any
+changed **test** files for high-precision flakiness anti-patterns — hard-coded
+sleeps / fixed waits (`time.Sleep`, `Thread.sleep`, `Task.Delay`,
+`asyncio.sleep`, `*.waitForTimeout`, numeric `cy.wait`, `usleep`, `sleep(<n>)`)
+and unseeded randomness (`Math.random`, Python `random.*`, Go `math/rand`).
+Matches merge into the normal findings, so they render, count toward
+`--fail-on`, and `--copy` like any other finding — but they are **deterministic
+and reproducible**: no model call, no JSON-schema change. On by default for the
+API/mock providers; turn it off per-run with `--no-flaky` or persistently with
+`review.flaky: false`. CLI-tool-backed plain-text providers are unaffected for
+now.
 
 ### `commitbrief commit`
 
@@ -493,6 +507,8 @@ command:
 commit:
   type: plain                      # default --type for `commitbrief commit` (plain|conventional|conventional+body|gitmoji|subject+body)
   generate: 1                      # default --generate (number of message alternatives)
+review:
+  flaky: true                      # deterministic flaky-test detector pre-pass (ADR-0022); --no-flaky overrides per-run
 ```
 
 ### Default command (`command.default`)

internal/cli/config.go

@@ -225,12 +225,23 @@ func configFieldGet(cfg *config.Config, path string) (string, error) {
 			return "", fmt.Errorf("config: unknown field %q in command (allowed: default)", parts[1])
 		}
 
+	case "review":
+		if len(parts) != 2 {
+			return "", fmt.Errorf("config: %q must be review.<field>", path)
+		}
+		switch parts[1] {
+		case "flaky":
+			return strconv.FormatBool(cfg.Review.Flaky), nil
+		default:
+			return "", fmt.Errorf("config: unknown field %q in review (allowed: flaky)", parts[1])
+		}
+
 	case "version":
 		// Read-only via get; explicitly rejected by configFieldSet.
 		return strconv.Itoa(cfg.Version), nil
 
 	default:
-		return "", fmt.Errorf("config: unknown top-level field %q (allowed: provider, providers.*, output.*, cache.*, guard.*, cost.*, command.*, version)", parts[0])
+		return "", fmt.Errorf("config: unknown top-level field %q (allowed: provider, providers.*, output.*, cache.*, guard.*, cost.*, command.*, review.*, version)", parts[0])
 	}
 }
 
@@ -392,11 +403,27 @@ func configFieldSet(cfg *config.Config, path, value string) error {
 		}
 		return nil
 
+	case "review":
+		if len(parts) != 2 {
+			return fmt.Errorf("config: %q must be review.<field>", path)
+		}
+		switch parts[1] {
+		case "flaky":
+			b, err := parseConfigBool(value)
+			if err != nil {
+				return fmt.Errorf("config: review.flaky: %w", err)
+			}
+			cfg.Review.Flaky = b
+		default:
+			return fmt.Errorf("config: unknown field %q in review (allowed: flaky)", parts[1])
+		}
+		return nil
+
 	case "version":
 		return errors.New("config: version is managed by migrations and cannot be set manually")
 
 	default:
-		return fmt.Errorf("config: unknown top-level field %q (allowed: provider, providers.*, output.*, cache.*, guard.*, cost.*, command.*)", parts[0])
+		return fmt.Errorf("config: unknown top-level field %q (allowed: provider, providers.*, output.*, cache.*, guard.*, cost.*, command.*, review.*)", parts[0])
 	}
 }
 

internal/cli/flaky_merge_test.go

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package cli
+
+import (
+	"testing"
+
+	"github.com/CommitBrief/commitbrief/internal/render"
+)
+
+func TestMergeFlaky(t *testing.T) {
+	llm := []render.Finding{
+		{Severity: render.SeverityHigh, File: "a.go", Line: 10, Title: "llm-1"},
+		{Severity: render.SeverityLow, File: "b.go", Line: 20, Title: "llm-2"},
+	}
+	flaky := []render.Finding{
+		// same file:line as llm-1 → dropped (the model already covered the line)
+		{Severity: render.SeverityMedium, File: "a.go", Line: 10, Title: "flaky-dup"},
+		// unique line → kept
+		{Severity: render.SeverityMedium, File: "c.go", Line: 5, Title: "flaky-new"},
+	}
+
+	got := mergeFlaky(llm, flaky)
+	if len(got) != 3 {
+		t.Fatalf("len = %d, want 3 (2 llm + 1 unique flaky): %+v", len(got), got)
+	}
+	// Every LLM finding is preserved, in order, ahead of the flaky ones.
+	if got[0].Title != "llm-1" || got[1].Title != "llm-2" {
+		t.Errorf("LLM findings not preserved in order: %+v", got[:2])
+	}
+	for _, f := range got {
+		if f.Title == "flaky-dup" {
+			t.Errorf("flaky finding at an LLM-occupied line should be dropped: %+v", f)
+		}
+	}
+	if got[2].Title != "flaky-new" {
+		t.Errorf("unique flaky finding should be appended last; got %+v", got[2])
+	}
+}
+
+func TestMergeFlaky_Empty(t *testing.T) {
+	llm := []render.Finding{{Severity: render.SeverityInfo, File: "a.go", Line: 1, Title: "x"}}
+	// No flaky findings → llm returned unchanged (the common + plain-text path).
+	if got := mergeFlaky(llm, nil); len(got) != 1 || got[0].Title != "x" {
+		t.Errorf("mergeFlaky(llm, nil) should return llm unchanged; got %+v", got)
+	}
+	// No LLM findings (clean review) + flaky present → flaky surfaced.
+	flaky := []render.Finding{{Severity: render.SeverityMedium, File: "t_test.go", Line: 3, Title: "f"}}
+	if got := mergeFlaky(nil, flaky); len(got) != 1 || got[0].Title != "f" {
+		t.Errorf("mergeFlaky(nil, flaky) should return flaky; got %+v", got)
+	}
+}

internal/cli/review.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -21,6 +22,7 @@ import (
 	"github.com/CommitBrief/commitbrief/internal/clipboard"
 	"github.com/CommitBrief/commitbrief/internal/config"
 	"github.com/CommitBrief/commitbrief/internal/diff"
+	"github.com/CommitBrief/commitbrief/internal/flaky"
 	"github.com/CommitBrief/commitbrief/internal/git"
 	"github.com/CommitBrief/commitbrief/internal/guard"
 	"github.com/CommitBrief/commitbrief/internal/ignore"
@@ -233,6 +235,16 @@ func runReview(cmd *cobra.Command, scope reviewScopeFlags, diffArgs []string) er
 		return showPromptOutput(cmd, p)
 	}
 
+	// Deterministic flaky-test pre-pass (ADR-0022). Static, provider-free
+	// findings from the filtered diff, merged into the structured results in
+	// both the cache-hit and fresh paths below. Skipped for plain-text/CLI
+	// providers (their output isn't structured) and when disabled via
+	// review.flaky=false or --no-flaky.
+	var flakyFindings []render.Finding
+	if app.Config.Review.Flaky && !global.noFlaky && !plainText {
+		flakyFindings = flaky.New(app.Catalog).Detect(parsed)
+	}
+
 	model := app.Config.Providers[app.Config.Provider].Model
 	if model == "" {
 		model = prov.DefaultModel()
@@ -289,6 +301,7 @@ func runReview(cmd *cobra.Command, scope reviewScopeFlags, diffArgs []string) er
 			case cache.FormatJSON, "":
 				findings, _ = render.ParseFindings(entry.Result.Content)
 			}
+			findings = mergeFlaky(findings, flakyFindings)
 			if entry.Result.Format == cache.FormatPlainText {
 				// CLI-emitted output: stream the cached body verbatim to
 				// stdout instead of going through the cards renderer.
@@ -407,6 +420,7 @@ func runReview(cmd *cobra.Command, scope reviewScopeFlags, diffArgs []string) er
 	case cache.FormatMarkdownFallback:
 		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), app.Catalog.T("review.degraded"))
 	}
+	findings = mergeFlaky(findings, flakyFindings)
 
 	respModel := model
 	meta := render.Meta{
@@ -466,6 +480,31 @@ func runReview(cmd *cobra.Command, scope reviewScopeFlags, diffArgs []string) er
 	return applyFailOn(cmd, app, findings)
 }
 
+// mergeFlaky appends deterministic flaky-test findings (ADR-0022) to the LLM
+// findings, skipping any whose (file, line) the LLM already reported so the
+// same line isn't surfaced twice. Every LLM finding is preserved; flaky
+// findings are additive. Returns llm unchanged when there are no flaky
+// findings — the common case, and the plain-text/CLI path where the detector
+// never ran.
+func mergeFlaky(llm, flakyFindings []render.Finding) []render.Finding {
+	if len(flakyFindings) == 0 {
+		return llm
+	}
+	seen := make(map[string]struct{}, len(llm))
+	for _, f := range llm {
+		seen[f.File+":"+strconv.Itoa(f.Line)] = struct{}{}
+	}
+	out := make([]render.Finding, 0, len(llm)+len(flakyFindings))
+	out = append(out, llm...)
+	for _, f := range flakyFindings {
+		if _, dup := seen[f.File+":"+strconv.Itoa(f.Line)]; dup {
+			continue
+		}
+		out = append(out, f)
+	}
+	return out
+}
+
 // emitPlainText streams a CLI-provider's already-formatted output
 // verbatim. We don't run it through the cards renderer or glamour —
 // the host CLI's output is the final form the user wants to see, and

internal/cli/root.go

@@ -31,6 +31,7 @@ type globalFlags struct {
 	compact       bool
 	allowSecrets  bool
 	noCostCheck   bool
+	noFlaky       bool
 	copy          bool
 	suggestCommit bool
 	commitType    string // commit: --type <format>; "" → commit.type config → "plain"
@@ -95,6 +96,7 @@ func newRootCmd() *cobra.Command {
 	flags.BoolVar(&global.compact, "compact", false, "one-line per finding (dense review output)")
 	flags.BoolVar(&global.allowSecrets, "allow-secrets", false, "bypass the pre-send secret scanner (use with care)")
 	flags.BoolVar(&global.noCostCheck, "no-cost-check", false, "skip the pre-send cost estimate prompt")
+	flags.BoolVar(&global.noFlaky, "no-flaky", false, "skip the deterministic flaky-test detector (ADR-0022)")
 	flags.BoolVar(&global.copy, "copy", false, "copy findings (severity, path, title, description) to the system clipboard via OSC 52 + native tool")
 	flags.BoolVar(&global.suggestCommit, "suggest-commit", false, "after the review, suggest a Conventional Commit message for the staged diff (requires --staged; prints to stdout; not with --json/--markdown/--output)")
 	flags.StringVar(&global.failOn, "fail-on", "", "exit 1 if any finding meets/exceeds severity (critical|high|medium|low|info|any|none)")

internal/config/config.go

@@ -14,6 +14,16 @@ type Config struct {
 	Cost      CostConfig                `yaml:"cost"`
 	Command   CommandConfig             `yaml:"command"`
 	Commit    CommitConfig              `yaml:"commit"`
+	Review    ReviewConfig              `yaml:"review"`
+}
+
+// ReviewConfig toggles review-time behaviors that aren't pre-send guards.
+// Flaky enables the deterministic static flaky-test detector (ADR-0022): a
+// provider-free pre-pass that flags timing/randomness anti-patterns in
+// changed test files and merges them into the structured findings. On by
+// default; precedence is --no-flaky > review.flaky config > built-in (true).
+type ReviewConfig struct {
+	Flaky bool `yaml:"flaky"`
 }
 
 // CommitConfig sets defaults for the `commit` command (ADR-0019) so a repo

internal/config/defaults.go

@@ -31,5 +31,8 @@ func Default() *Config {
 			Type:     "plain",
 			Generate: 1,
 		},
+		Review: ReviewConfig{
+			Flaky: true,
+		},
 	}
 }