diff --git a/docs/contributing/TECHNICAL.md b/docs/contributing/TECHNICAL.md index 54f15422ae..c13f3d4fd0 100644 --- a/docs/contributing/TECHNICAL.md +++ b/docs/contributing/TECHNICAL.md @@ -306,7 +306,7 @@ Start here, then drill down into each README for file-level details. | [`discover/`](../src/discover/README.md) | History analysis + rewrite registry | Rewrite patterns, session providers, compound command splitting | | [`learn/`](../src/learn/README.md) | CLI correction detection | Error classification, correction pair detection, rule generation | | [`parser/`](../src/parser/README.md) | Parser infrastructure | Canonical types (TestResult, LintResult, etc.), 3-tier format modes, migration guide | -| [`filters/`](../src/filters/README.md) | TOML filter configs | TOML DSL syntax, 8-stage pipeline, inline testing, naming conventions | +| [`filters/`](../src/filters/README.md) | TOML filter configs | TOML DSL syntax, 9-stage pipeline, inline testing, naming conventions | ### `hooks/` — Deployed hook artifacts (root directory) @@ -353,7 +353,7 @@ Compiled filter modules for complex transformations, cutting 60-95% of the bash ### TOML DSL Filters (src/filters/*.toml) -Declarative filters with an 8-stage pipeline: strip ANSI, regex replace, match output, strip/keep lines, truncate lines, head/tail, max lines, on-empty message. Loaded from three tiers: built-in (compiled), global (`~/.config/rtk/filters/`), project-local (`.rtk/filters/`, trust-gated). +Declarative filters with a 9-stage pipeline: strip ANSI, regex replace, match output, strip/keep lines, truncate lines, collapse repeats, head/tail, max lines, on-empty message. Loaded from three tiers: built-in (compiled), global (`~/.config/rtk/filters/`), project-local (`.rtk/filters/`, trust-gated). > **Details**: [`src/core/README.md`](../src/core/README.md) covers the TOML filter engine. diff --git a/src/core/README.md b/src/core/README.md index ab3529ef59..b13efe8400 100644 --- a/src/core/README.md +++ b/src/core/README.md @@ -15,16 +15,17 @@ Core infrastructure shared by all RTK command modules. Every filter, tracker, an ## TOML Filter Pipeline -The TOML DSL applies 8 stages in order: +The TOML DSL applies 9 stages in order: 1. **strip_ansi**: Remove ANSI escape codes if enabled 2. **replace**: Line-by-line regex substitutions (chainable, supports backreferences) 3. **match_output**: Short-circuit rules (if output matches pattern, return message; `unless` field prevents swallowing errors) 4. **strip/keep_lines**: Filter lines by regex (mutually exclusive) 5. **truncate_lines_at**: Truncate each line to N chars (unicode-safe) -6. **head/tail_lines**: Keep first N or last N lines (with omit message) -7. **max_lines**: Absolute line cap applied after head/tail -8. **on_empty**: Return message if result is empty after all stages +6. **collapse_repeats**: Keep the first occurrence of each distinct line, append `(×N)` to repeats (opt-in, `keep_tail` lines exempt) +7. **head/tail_lines**: Keep first N or last N lines (with omit message) +8. **max_lines**: Absolute line cap applied after head/tail +9. **on_empty**: Return message if result is empty after all stages Three-tier filter lookup (first match wins): 1. `.rtk/filters.toml` (project-local, requires `rtk trust`) diff --git a/src/core/toml_filter.rs b/src/core/toml_filter.rs index 977e8d974b..2ba139631b 100644 --- a/src/core/toml_filter.rs +++ b/src/core/toml_filter.rs @@ -19,13 +19,14 @@ /// 3. match_output — short-circuit: if blob matches a pattern, return message immediately /// 4. strip/keep_lines — filter lines by regex /// 5. truncate_lines_at — truncate each line to N chars -/// 6. head/tail_lines — keep first/last N lines -/// 7. max_lines — absolute line cap -/// 8. on_empty — message if result is empty +/// 6. collapse_repeats — collapse repeated identical lines, rendering the count back in +/// 7. head/tail_lines — keep first/last N lines +/// 8. max_lines — absolute line cap +/// 9. on_empty — message if result is empty use super::constants::RTK_META_COMMANDS; use regex::{Regex, RegexSet}; use serde::Deserialize; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::LazyLock; // Built-in filters: concatenated from src/filters/*.toml by build.rs at compile time. @@ -59,6 +60,16 @@ struct ReplaceRule { replacement: String, } +/// Opt-in repeated-line collapse. Absent means disabled, so existing filters +/// keep their behaviour. `keep_tail` trailing lines are exempt and emitted +/// verbatim — test runners put their summary last. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CollapseRepeats { + #[serde(default)] + keep_tail: usize, +} + /// An inline test case attached to a filter in the TOML. /// Lives in `[[tests.]]` sections, separate from `[filters.*]`. #[derive(Deserialize)] @@ -98,6 +109,7 @@ struct TomlFilterDef { #[serde(default)] keep_lines_matching: Vec, truncate_lines_at: Option, + collapse_repeats: Option, head_lines: Option, tail_lines: Option, max_lines: Option, @@ -145,6 +157,7 @@ pub struct CompiledFilter { match_output: Vec, line_filter: LineFilter, truncate_lines_at: Option, + collapse_repeats: Option, head_lines: Option, tail_lines: Option, pub max_lines: Option, @@ -383,6 +396,7 @@ fn compile_filter(name: String, def: TomlFilterDef) -> Result( /// 3. match_output — short-circuit if blob matches a pattern /// 4. strip/keep_lines — filter lines by regex /// 5. truncate_lines_at — truncate each line to N chars -/// 6. head/tail_lines — keep first/last N lines -/// 7. max_lines — absolute line cap -/// 8. on_empty — message if result is empty +/// 6. collapse_repeats — collapse repeated identical lines with a count +/// 7. head/tail_lines — keep first/last N lines +/// 8. max_lines — absolute line cap +/// 9. on_empty — message if result is empty pub fn apply_filter(filter: &CompiledFilter, stdout: &str) -> String { apply_filter_with_info(filter, stdout).0 } @@ -573,12 +588,17 @@ pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, .collect(); } + // 6. collapse_repeats — opt-in; distinct lines all survive, so nothing is lost + if let Some(collapse) = &filter.collapse_repeats { + lines = collapse_repeated_lines(lines, collapse.keep_tail); + } + let snapshot_for_tail = !intra_line_loss && filter.tail_lines.is_none() && (filter.head_lines.is_some() || filter.max_lines.is_some()); let pre_cut = snapshot_for_tail.then(|| lines.clone()); - // 6. head + tail + // 7. head + tail let total = lines.len(); let mut noncontiguous_drop = false; let mut head_cut: Option = None; @@ -605,7 +625,7 @@ pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, } } - // 7. max_lines — absolute cap applied after head/tail (includes omit messages) + // 8. max_lines — absolute cap applied after head/tail (includes omit messages) let mut max_cut: Option = None; if let Some(max) = filter.max_lines { if lines.len() > max { @@ -616,7 +636,7 @@ pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, } } - // 8. on_empty + // 9. on_empty let result = lines.join("\n"); if result.trim().is_empty() { if let Some(ref msg) = filter.on_empty { @@ -646,6 +666,40 @@ pub fn apply_filter_with_info(filter: &CompiledFilter, stdout: &str) -> (String, (result, loss) } +/// Keep the first occurrence of every distinct line and append `(×N)` to the +/// ones that repeat. Duplicates in this kind of output are scattered rather than +/// consecutive, so collapsing per run barely helps; collapsing globally keeps +/// every distinct line, in first-occurrence order, and only drops exact repeats. +/// +/// The last `keep_tail` lines are exempt: they are emitted verbatim, and they do +/// not feed the counts, so a trailing summary survives untouched. +fn collapse_repeated_lines(lines: Vec, keep_tail: usize) -> Vec { + if lines.len() <= keep_tail { + return lines; + } + let (head, tail) = lines.split_at(lines.len() - keep_tail); + + let mut counts: HashMap<&str, usize> = HashMap::new(); + for line in head { + *counts.entry(line.as_str()).or_insert(0) += 1; + } + + let mut seen: HashSet<&str> = HashSet::with_capacity(counts.len()); + let mut out = Vec::with_capacity(counts.len() + keep_tail); + for line in head { + if !seen.insert(line.as_str()) { + continue; + } + // A count on an invisible line is noise — blank lines just collapse. + match counts[line.as_str()] { + n if n > 1 && !line.trim().is_empty() => out.push(format!("{} (×{})", line, n)), + _ => out.push(line.clone()), + } + } + out.extend_from_slice(tail); + out +} + // --------------------------------------------------------------------------- // rtk verify — inline test execution // --------------------------------------------------------------------------- @@ -818,6 +872,78 @@ mod tests { .expect("expected at least one filter") } + // --- collapse_repeats (stage 6) --- + + fn collapsing_filter(keep_tail: usize) -> CompiledFilter { + first_filter(&format!( + "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\ncollapse_repeats = {{ keep_tail = {keep_tail} }}\n" + )) + } + + #[test] + fn collapse_repeats_is_off_unless_configured() { + let toml = "schema_version = 1\n[filters.f]\nmatch_command = \"^cmd\"\n"; + assert_eq!(apply_filter(&first_filter(toml), "a\na\nb"), "a\na\nb"); + } + + #[test] + fn collapse_repeats_counts_globally_not_per_run() { + // Duplicates are scattered, so a consecutive-run collapse would change nothing. + let out = apply_filter(&collapsing_filter(0), "a\nb\na\nc\na\nb"); + assert_eq!(out, "a (×3)\nb (×2)\nc"); + } + + #[test] + fn collapse_repeats_keeps_tail_verbatim() { + let out = apply_filter(&collapsing_filter(2), "ok\nok\nok\nok"); + assert_eq!(out, "ok (×2)\nok\nok"); + } + + #[test] + fn collapse_repeats_leaves_blank_lines_uncounted() { + let out = apply_filter(&collapsing_filter(0), "a\n\nb\n\na"); + assert_eq!(out, "a (×2)\n\nb"); + } + + #[test] + fn collapse_repeats_reports_no_loss() { + let (_, loss) = apply_filter_with_info(&collapsing_filter(0), "a\na\nb"); + assert_eq!(loss, Lossiness::None); + } + + #[test] + fn collapse_repeats_saves_tokens_on_repetitive_output() { + let mut input = String::new(); + for i in 0..300 { + input.push_str("[trace] gc cycle\n"); + input.push_str("[trace] allocating buffer\n"); + if i % 100 == 0 { + input.push_str(&format!("checkpoint {i}\n")); + } + } + input.push_str("3211 assertions, 0 failures\n"); + + let output = apply_filter(&collapsing_filter(5), &input); + let count_tokens = |s: &str| s.split_whitespace().count(); + let savings = 100.0 - (count_tokens(&output) as f64 / count_tokens(&input) as f64 * 100.0); + assert!(savings >= 60.0, "expected >=60% savings, got {savings:.1}%"); + assert!(output.ends_with("3211 assertions, 0 failures")); + } + + #[test] + fn builtin_lua_filters_route_by_command() { + let filters = make_filters(BUILTIN_TOML); + let name_of = |cmd: &str| find_filter_in(cmd, &filters).map(|f| f.name.as_str()); + + assert_eq!(name_of("luajit tests/run.lua"), Some("lua")); + assert_eq!(name_of("lua5.4 tests/run.lua"), Some("lua")); + assert_eq!(name_of("luacheck ."), Some("luacheck")); + // Bare REPL and the compiler stay on the raw passthrough path. + assert_eq!(name_of("lua"), None); + assert_eq!(name_of("luac -p src/init.lua"), None); + assert_eq!(name_of("luac -o out.luac src/init.lua"), None); + } + #[test] fn command_matches_filter_agrees_with_find_matching_filter() { for cmd in ["jj log", "jq .", "frobnicate xyz", "cd /tmp"] { @@ -1845,6 +1971,8 @@ match_command = "^make\\b" "helm", "iptables", "liquibase", + "lua", + "luacheck", "make", "markdownlint", "mix-compile", @@ -1892,8 +2020,8 @@ match_command = "^make\\b" let filters = make_filters(BUILTIN_TOML); assert_eq!( filters.len(), - 63, - "Expected exactly 63 built-in filters, got {}. \ + 65, + "Expected exactly 65 built-in filters, got {}. \ Update this count when adding/removing filters in src/filters/.", filters.len() ); @@ -1950,11 +2078,11 @@ expected = "output line 1\noutput line 2" let combined = format!("{}\n\n{}", BUILTIN_TOML, new_filter); let filters = make_filters(&combined); - // All 63 existing filters still present + 1 new = 64 + // All 65 existing filters still present + 1 new = 66 assert_eq!( filters.len(), - 64, - "Expected 64 filters after concat (63 built-in + 1 new)" + 66, + "Expected 66 filters after concat (65 built-in + 1 new)" ); // New filter is discoverable diff --git a/src/discover/rules.rs b/src/discover/rules.rs index 49c0ff740a..a5ab6a7f67 100644 --- a/src/discover/rules.rs +++ b/src/discover/rules.rs @@ -849,6 +849,24 @@ pub const RULES: &[RtkRule] = &[ savings_pct: 65.0, ..RtkRule::DEFAULT }, + RtkRule { + // Versioned binaries (lua5.4) have no rewrite_prefix, so leave them to + // the TOML registry fallback rather than claiming them here. + pattern: r"^(?:luajit|lua)\s+\S", + rtk_cmd: "rtk lua", + rewrite_prefixes: &["luajit", "lua"], + category: "Build", + savings_pct: 90.0, + ..RtkRule::DEFAULT + }, + RtkRule { + pattern: r"^luacheck\b", + rtk_cmd: "rtk luacheck", + rewrite_prefixes: &["luacheck"], + category: "Build", + savings_pct: 90.0, + ..RtkRule::DEFAULT + }, RtkRule { pattern: r"^shellcheck\b", rtk_cmd: "rtk shellcheck", diff --git a/src/filters/README.md b/src/filters/README.md index 5b9bbb946e..5e05174b0d 100644 --- a/src/filters/README.md +++ b/src/filters/README.md @@ -57,10 +57,41 @@ expected = "expected filtered output" | `replace` | array | Regex substitutions (`{ pattern, replacement }`) | | `match_output` | array | Short-circuit rules (`{ pattern, message }`) | | `truncate_lines_at` | int | Truncate lines longer than N characters | +| `collapse_repeats` | table | Opt-in: collapse repeated identical lines (see below) | | `max_lines` | int | Keep only the first N lines | | `tail_lines` | int | Keep only the last N lines (applied after other filters) | | `on_empty` | string | Fallback message when filtered output is empty | +## Collapsing repeated lines + +Some commands emit the same line hundreds of times — a Lua test suite that +`print()`s from the code under test is the usual case. `collapse_repeats` keeps +the first occurrence of every distinct line, in order, and renders the repeat +count back in: + +```toml +[filters.lua] +match_command = "^(luajit|lua)[0-9.]*\\s+\\S" +collapse_repeats = { keep_tail = 5 } +``` + +``` +[trace] gc cycle (×802) +loading fixtures +3211 assertions, 0 failures +``` + +Duplicates in this kind of output are scattered rather than consecutive, so the +count is global (first occurrence wins), not per run. The last `keep_tail` lines +are exempt — they are emitted verbatim and do not feed the counts, so a trailing +summary survives even when it repeats an earlier line. When it does repeat one, +both are visible: the earlier occurrence carries the count, the tail copy is +printed bare and is not counted in it. + +The field is **opt-in**: a filter that doesn't set it behaves exactly as before. +Only enable it for commands whose output is program chatter. Tools that repeat +lines on purpose (per-row results, a diff, a table) must not use it. + ## Naming convention Use the command name as the filename: `terraform-plan.toml`, `docker-inspect.toml`, `mix-compile.toml`. @@ -92,7 +123,7 @@ flowchart TD R["TomlFilterRegistry::load()\n1. .rtk/filters.toml\n2. ~/.config/rtk/filters.toml\n3. BUILTIN_TOML\n4. passthrough"] --> S S{"match_command\nmatches?"} -->|"no match"| T[["exec raw (passthrough)"]] S -->|"match"| U["exec command\ncapture stdout"] - U --> V["8-stage pipeline\nstrip_ansi → replace → match_output\n→ strip/keep_lines → truncate\n→ tail_lines → max_lines → on_empty"] + U --> V["9-stage pipeline\nstrip_ansi → replace → match_output\n→ strip/keep_lines → truncate\n→ collapse_repeats → tail_lines\n→ max_lines → on_empty"] V --> W[["print filtered output + exit code"]] end diff --git a/src/filters/lua.toml b/src/filters/lua.toml new file mode 100644 index 0000000000..63eb428f04 --- /dev/null +++ b/src/filters/lua.toml @@ -0,0 +1,39 @@ +[filters.lua] +description = "Collapse repeated lines in lua/luajit program output, keeping the last lines verbatim" +match_command = "^(luajit|lua)[0-9.]*\\s+\\S" +collapse_repeats = { keep_tail = 5 } + +[[tests.lua]] +name = "repeated chatter collapses, trailing summary survives" +input = """ +[trace] gc cycle +[trace] gc cycle +[trace] gc cycle +[trace] gc cycle +loading fixtures +[trace] gc cycle +[trace] gc cycle +running suite 1 +running suite 2 +running suite 3 + +3211 assertions, 0 failures +""" +expected = "[trace] gc cycle (×6)\nloading fixtures\nrunning suite 1\nrunning suite 2\nrunning suite 3\n\n3211 assertions, 0 failures" + +[[tests.lua]] +name = "output without repeats is unchanged" +input = """ +opening db +seeding 40 rows +running migrations +checking indexes +vacuuming +done +""" +expected = "opening db\nseeding 40 rows\nrunning migrations\nchecking indexes\nvacuuming\ndone" + +[[tests.lua]] +name = "empty input passes through" +input = "" +expected = "" diff --git a/src/filters/luacheck.toml b/src/filters/luacheck.toml new file mode 100644 index 0000000000..77454c84b6 --- /dev/null +++ b/src/filters/luacheck.toml @@ -0,0 +1,48 @@ +[filters.luacheck] +description = "Drop luacheck's per-file OK lines, keep every diagnostic and the Total footer" +match_command = "^luacheck\\b" +strip_ansi = true +strip_lines_matching = [ + "^Checking\\s+\\S.*\\bOK\\s*$", + "^\\s*$", +] + +[[tests.luacheck]] +name = "clean files dropped, warnings and footer kept" +input = """ +Checking AutoLog.lua OK +Checking BuffChecker.lua 2 warnings + + BuffChecker.lua:12:11: unused variable 'ctx' + BuffChecker.lua:20:1: line contains trailing whitespace + +Checking Core.lua OK + +Total: 2 warnings / 0 errors in 3 files +""" +expected = "Checking BuffChecker.lua 2 warnings\n BuffChecker.lua:12:11: unused variable 'ctx'\n BuffChecker.lua:20:1: line contains trailing whitespace\nTotal: 2 warnings / 0 errors in 3 files" + +[[tests.luacheck]] +name = "clean project collapses to the footer" +input = """ +Checking a.lua OK +Checking b.lua OK + +Total: 0 warnings / 0 errors in 2 files +""" +expected = "Total: 0 warnings / 0 errors in 2 files" + +[[tests.luacheck]] +name = "syntax errors are kept" +input = """ +Checking a.lua OK +Checking broken.lua Syntax error + +Total: 0 warnings / 1 error in 2 files +""" +expected = "Checking broken.lua Syntax error\nTotal: 0 warnings / 1 error in 2 files" + +[[tests.luacheck]] +name = "empty input passes through" +input = "" +expected = ""