feat(cmdfilter): port rtk's filter coverage (3 -> 24) and fix three DSL loss-typing bugs

THIRD-PARTY-NOTICES

@@ -0,0 +1,54 @@
+THIRD-PARTY NOTICES
+===================
+
+context-guru includes third-party material. This file records the notices those
+works require. It covers material adapted INTO this source tree; Go module
+dependencies carry their own licenses in the module cache and in go.sum.
+
+-------------------------------------------------------------------------------
+rtk (Rust Token Killer)
+-------------------------------------------------------------------------------
+
+Project:   rtk
+Homepage:  https://github.com/rtk-ai/rtk
+License:   Apache License, Version 2.0
+Notice:    Copyright 2024 rtk-ai and rtk-ai Labs
+
+Portions of this software are adapted from rtk, with modifications:
+
+  - components/dsl (DSL filter pipeline). The declarative filter engine — the
+    eight-stage pipeline (strip_ansi, replace, match_output+unless, strip/keep
+    lines, truncate_lines_at, head/tail, max_lines, on_empty), the field names,
+    the Lossiness typing, and the shared truncation cap classes (CAP_ERRORS,
+    CAP_WARNINGS, CAP_LIST, CAP_INVENTORY) — is adapted from rtk's TOML filter
+    DSL (src/core/toml_filter.rs, src/core/truncate.rs) and re-implemented in Go.
+
+  - components/offload (cmdfilter filter definitions and test corpora). The
+    shipped filter set is adapted from rtk's src/filters/*.toml, including its
+    inline test cases.
+
+Modifications made:
+
+  - Re-implemented in Go, from Rust.
+  - Filter documents are YAML with a schema_version, not TOML.
+  - Every filter's selector is rewritten. rtk matches a shell COMMAND string
+    (match_command); context-guru is a proxy and never sees the command, so each
+    filter matches an output-shape signature against the first non-empty line of
+    the tool output instead. rtk's command regexes are not portable as written.
+  - Filters whose outputs are indistinguishable from the output alone are merged
+    (terraform/tofu plan; terraform/tofu init; the pulumi subcommands).
+  - Every success-collapse (match_output) rule carries an `unless` guard; most of
+    rtk's do not. A proxy's agent cannot re-run a command to discover a warning
+    that a bare collapse swallowed.
+  - Per-filter line budgets are replaced by shared `cap` budget classes, applying
+    rtk's truncate.rs cap idea to the filter definitions (rtk's own TOML filters
+    hand-pick a max_lines each).
+  - truncate_lines_at records intra-line loss and marks the cut with an ellipsis.
+  - A `buildlog` cap class, a `family` field (per-family metrics), a `priority`
+    field (selector ordering), load-time guardrails (duplicate names rejected,
+    inline tests executed at load), and a selector-miss ledger were added.
+  - rtk's filter_stderr, its command-detection rules, and its native Rust
+    (non-DSL) filters are not ported — a proxy has no analogue for them.
+
+The Apache-2.0 license text is available at http://www.apache.org/licenses/LICENSE-2.0
+and, since context-guru is itself Apache-2.0 licensed, in this repository's LICENSE.

components/component.go

@@ -123,6 +123,22 @@ type Ctx struct {
 	// -1 = unknown/first turn/cache off ⇒ no tail restriction. Only meaningful when
 	// CacheAware is true.
 	MaxCachedIdx int
+	// FilterStats receives cmdfilter's per-filter ledger (which command families pay
+	// off, and which output shapes matched nothing). nil = not recording.
+	FilterStats FilterStatsSink
+}
+
+// FilterStatsSink records cmdfilter's per-filter/per-family ledger. metrics.Aggregator
+// implements it; the pipeline depends only on this interface. Implementations must be
+// safe for concurrent use.
+type FilterStatsSink interface {
+	// FilterAct notes one applied filter: its family (builds/tests/iac/pkg/net/...),
+	// its name, the content key (so a compaction re-sent verbatim next turn is counted
+	// once), and the tokens saved.
+	FilterAct(family, filter, contentKey string, saved int)
+	// FilterMiss notes a selector that matched no filter — the ledger that says which
+	// filter is worth writing next (after rtk's parse_failures table).
+	FilterMiss(selector string)
 }
 
 // TailOnly reports whether a supersession/age-based offloader may mutate the message

components/dsl/dsl.go

@@ -36,10 +36,42 @@ const (
 	LossWhole                  // non-contiguous / whole-blob loss; only full retrieval recovers
 )
 
+// Caps are the shared line budgets a filter picks by SIGNAL DENSITY (`cap: errors`)
+// instead of hand-picking a max_lines per filter. The first four names and values
+// are rtk's (src/core/truncate.rs); `buildlog` is ours — a build/plan transcript is
+// mostly noise but the line that matters can sit anywhere in it, so it gets a
+// deliberately generous budget. One map tunes the whole filter set.
+var Caps = map[string]int{
+	"errors":    20, // most actionable, shown the most
+	"warnings":  10, // lower signal density than errors
+	"list":      20, // flat lists (packages, services): one line per item
+	"inventory": 50, // exhaustive lookups (installed packages, file listings)
+	"buildlog":  80, // full build/plan transcripts: verbose, signal is positional
+}
+
+// ReducedCap is rtk's `reduced` deviation helper: a cap lowered for a more verbose
+// data class, underflow-safe — a deviation can never empty the budget.
+func ReducedCap(cap, by int) int {
+	if by > 0 && by < cap {
+		return cap - by
+	}
+	return cap
+}
+
 // Def is a raw filter definition (from YAML). All fields except Match are optional.
 type Def struct {
-	Description        string        `yaml:"description"`
-	Match              string        `yaml:"match"` // regex against the selector key
+	Description string `yaml:"description"`
+	Match       string `yaml:"match"` // regex against the selector key
+	// Family groups filters for per-family metrics (builds, tests, iac, pkg, net, ...).
+	Family string `yaml:"family"`
+	// Priority orders matching: higher first, then by name. Absent (0) = today's
+	// behavior (name order). Use it to put a specific filter ahead of a generic one,
+	// which matters more here than in rtk because we match on output shape.
+	Priority int `yaml:"priority"`
+	// Cap selects a shared budget class from Caps instead of a literal MaxLines;
+	// CapReduce lowers it for an extra-verbose variant. MaxLines wins if both are set.
+	Cap                string        `yaml:"cap"`
+	CapReduce          int           `yaml:"cap_reduce"`
 	StripANSI          bool          `yaml:"strip_ansi"`
 	Replace            []ReplaceRule `yaml:"replace"`
 	MatchOutput        []MatchRule   `yaml:"match_output"`
@@ -111,7 +143,23 @@ func Compile(name string, d Def) (*Compiled, error) {
 	if len(d.StripLinesMatching) > 0 && len(d.KeepLinesMatching) > 0 {
 		return nil, fmt.Errorf("dsl: filter %q sets both strip_lines_matching and keep_lines_matching", name)
 	}
-	m, err := regexp.Compile(d.Match)
+	if d.Cap != "" {
+		base, ok := Caps[d.Cap]
+		if !ok {
+			return nil, fmt.Errorf("dsl: filter %q unknown cap %q", name, d.Cap)
+		}
+		if d.MaxLines == nil { // an explicit max_lines still wins
+			n := ReducedCap(base, d.CapReduce)
+			d.MaxLines = &n
+		}
+	} else if d.CapReduce != 0 {
+		return nil, fmt.Errorf("dsl: filter %q sets cap_reduce without cap", name)
+	}
+	// The selector spans a few leading lines, not one, so `^`/`$` in a match regex must
+	// mean "start/end of A line" rather than "of the whole selector". Without (?m) a
+	// filter anchored at ^ only matches output whose very FIRST line is its signature,
+	// which is exactly the output-framing dependence a multi-line selector removes.
+	m, err := regexp.Compile("(?m)" + d.Match)
 	if err != nil {
 		return nil, fmt.Errorf("dsl: filter %q match: %w", name, err)
 	}
@@ -189,20 +237,28 @@ func Apply(c *Compiled, input string) (string, Lossiness) {
 	} else if len(c.keepLines) > 0 {
 		lines = filterLines(lines, c.keepLines, true)
 	}
-	// 5. truncate_lines_at (unicode-safe per-line cap)
+	// 5. truncate_lines_at (unicode-safe per-line cap). An intra-line cut is a REAL
+	// loss and is non-contiguous by nature (every long line loses its own tail), so
+	// it types as LossWhole; and it appends an ellipsis, because a silent mid-line
+	// cut reads as corrupted output to a model (both after rtk).
+	loss := LossNone
 	if c.def.TruncateLinesAt != nil {
 		n := *c.def.TruncateLinesAt
 		for i, l := range lines {
-			if r := []rune(l); len(r) > n {
-				lines[i] = string(r[:n])
+			if t := truncateRunes(l, n); t != l {
+				lines[i] = t
+				loss = LossWhole
 			}
 		}
 	}
 
-	loss := LossNone
 	// 6. head/tail
 	if c.def.HeadLines != nil || c.def.TailLines != nil {
-		lines, loss = headTail(lines, c.def.HeadLines, c.def.TailLines)
+		var htLoss Lossiness
+		lines, htLoss = headTail(lines, c.def.HeadLines, c.def.TailLines)
+		if htLoss > loss { // keep the more severe of an intra-line cut and a line drop
+			loss = htLoss
+		}
 	}
 	// 7. max_lines (absolute cap, counts the omission marker)
 	if c.def.MaxLines != nil && len(lines) > *c.def.MaxLines {
@@ -212,7 +268,7 @@ func Apply(c *Compiled, input string) (string, Lossiness) {
 		if loss == LossNone {
 			loss = LossTail
 		} else {
-			loss = LossWhole
+			loss = LossWhole // already lossy above: the cap is no longer a clean tail cut
 		}
 	}
 	out := strings.Join(lines, "\n")
@@ -223,6 +279,20 @@ func Apply(c *Compiled, input string) (string, Lossiness) {
 	return out, loss
 }
 
+// truncateRunes caps a line at n runes, marking the cut with an ellipsis that fits
+// INSIDE the budget (so the result is never longer than n). Ported from rtk's
+// utils::truncate.
+func truncateRunes(s string, n int) string {
+	r := []rune(s)
+	if len(r) <= n {
+		return s
+	}
+	if n < 3 {
+		return "..."
+	}
+	return string(r[:n-3]) + "..."
+}
+
 func filterLines(lines []string, res []*regexp.Regexp, keep bool) []string {
 	out := lines[:0:0]
 	for _, l := range lines {
@@ -272,11 +342,27 @@ func headTail(lines []string, head, tail *int) ([]string, Lossiness) {
 	return lines, LossNone
 }
 
-// Registry holds compiled filters, matched first-by-sorted-name for determinism.
-type Registry struct{ filters []*Compiled }
+// Family is the filter's command family, used for per-family metrics. "" when unset.
+func (c *Compiled) Family() string {
+	if c.def.Family == "" {
+		return "other"
+	}
+	return c.def.Family
+}
+
+// Registry holds compiled filters, matched by descending priority then by name for
+// determinism (specific-before-generic without relying on alphabetical luck).
+type Registry struct {
+	filters []*Compiled
+	names   map[string]struct{}
+}
 
 // Load parses a YAML filter document and appends its filters to the registry.
-// schema_version must be 1. Filters are stored sorted by name.
+// schema_version must be 1. Duplicate filter names are rejected (a silently
+// shadowed filter is a debugging trap — rtk's build.rs rejects them too) and any
+// inline tests the document carries must pass, so a broken filter fails loudly at
+// load instead of quietly mangling output. (Requiring a test to EXIST is enforced
+// for the shipped builtins by a unit test, not here — user configs stay free.)
 func (r *Registry) Load(b []byte) error {
 	var f File
 	dec := yaml.NewDecoder(strings.NewReader(string(b)))
@@ -292,13 +378,31 @@ func (r *Registry) Load(b []byte) error {
 		names = append(names, n)
 	}
 	sort.Strings(names)
+	if r.names == nil {
+		r.names = map[string]struct{}{}
+	}
 	for _, n := range names {
+		if _, dup := r.names[n]; dup {
+			return fmt.Errorf("dsl: duplicate filter name %q", n)
+		}
 		c, err := Compile(n, f.Filters[n])
 		if err != nil {
 			return err
 		}
+		for _, tc := range f.Tests[n] {
+			if got, _ := Apply(c, tc.Input); !sameText(got, tc.Expected) {
+				return fmt.Errorf("dsl: filter %q test %q failed: got %q want %q", n, tc.Name, got, tc.Expected)
+			}
+		}
+		r.names[n] = struct{}{}
 		r.filters = append(r.filters, c)
 	}
+	sort.SliceStable(r.filters, func(i, j int) bool {
+		if r.filters[i].def.Priority != r.filters[j].def.Priority {
+			return r.filters[i].def.Priority > r.filters[j].def.Priority
+		}
+		return r.filters[i].Name < r.filters[j].Name
+	})
 	return nil
 }
 
@@ -312,6 +416,10 @@ func (r *Registry) Match(key string) *Compiled {
 	return nil
 }
 
+func sameText(got, want string) bool {
+	return strings.TrimRight(got, "\n") == strings.TrimRight(want, "\n")
+}
+
 // Len reports how many filters are loaded.
 func (r *Registry) Len() int { return len(r.filters) }
 
@@ -329,7 +437,7 @@ func RunTests(b []byte) (failures []string, err error) {
 		}
 		for _, tc := range f.Tests[name] {
 			got, _ := Apply(c, tc.Input)
-			if strings.TrimRight(got, "\n") != strings.TrimRight(tc.Expected, "\n") {
+			if !sameText(got, tc.Expected) {
 				failures = append(failures, name+"/"+tc.Name)
 			}
 		}