Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ moongrep scan --pattern 'inspect($_, content="true")' --output-json
```

JSON match records are written to standard output, one per line. Verbose
traversal messages and parse warnings are written to standard error. When no
traversal messages and scan warnings are written to standard error. When no
match is found, JSON mode writes nothing to standard output.

## Pattern Guards
Expand Down
2 changes: 1 addition & 1 deletion SKILL_CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ user.name == other.name
moongrep scan --pattern 'inspect($_, content="true")' --output-json
```

JSON 匹配记录会逐行写入标准输出。详细遍历信息和解析 warning 会写入标准错误;
JSON 匹配记录会逐行写入标准输出。详细遍历信息和扫描 warning 会写入标准错误;
JSON 模式下没有命中时,标准输出为空。

## 模式附加条件
Expand Down
18 changes: 0 additions & 18 deletions TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,6 @@
an explicit kind for cross-position reuse, improving the diagnostic, or
adding a storage-path metavariable kind.

- Correct the commands in the embedded rule-writing guides. They currently
recommend `moon run . -- scan ...`, which only runs moongrep when invoked
from the moongrep module. Document the installed `moongrep scan ...` and
Mooncakes/Wasm `moonx moonbit-community/moongrep -- scan ...` forms, and
keep `moon run .` in a contributor-only section if it is still needed.

- Define and document the contract for `#moongrep.skip`. It is currently
recognized on functions, impls, top-level lets, tests, and views; ignores
any payload, including `false`; and suppresses structural rules but not
taint analysis. If it remains public, cover those semantics in the user and
e2e documentation. Otherwise, make it internal or remove it.

- Clarify constant matching in the RuleSpec overview. Constants are compared
using the parser AST constant kind and preserved source spelling, so
equivalent values such as `1000` and `1_000` do not necessarily match.
Update the English and Chinese summaries so they do not imply that all
formatting is ignored.

- Make the Chinese rule documentation available through the `docs` command,
or document that only English documents are embedded. `docs/export.mbt`
currently registers only `RuleSpec` and `WritingRules`; consider adding
Expand Down
4 changes: 2 additions & 2 deletions cli/json_render.mbt
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ fn render_scan_hit_json(hit : ScanHit) -> String {
}

///|
// JSON stdout is finding-only: scan trace, parse warnings, and the human
// JSON stdout is finding-only: scan trace, scan warnings, and the human
// no-match summary are omitted so every non-empty line is a machine-readable
// finding record. Parse warnings are written separately to stderr.
// finding record. Scan warnings are written separately to stderr.
fn render_directory_scan_json(result : DirectoryScanResult) -> String {
let records : Array[String] = []
for hit in result.hits {
Expand Down
12 changes: 8 additions & 4 deletions cli/json_render_wbtest.mbt
Original file line number Diff line number Diff line change
Expand Up @@ -84,17 +84,21 @@ test "json directory output contains only one line per finding" {
render_directory_scan_json({
scan_trace: ["trace"],
hits: [],
skipped_files: [
{ file: "bad.mbt", block_start_line: None, reason: "parse failed" },
warnings: [
Parse({
file: "bad.mbt",
block_start_line: None,
reason: "parse failed",
}),
],
}),
content="",
)
let rendered = render_directory_scan_json({
scan_trace: ["trace"],
hits: [render_test_hit(3, 3), render_test_hit(8, 8)],
skipped_files: [
{ file: "bad.mbt", block_start_line: None, reason: "parse failed" },
warnings: [
Parse({ file: "bad.mbt", block_start_line: None, reason: "parse failed" }),
],
})
let records = rendered.split("\n").to_array()
Expand Down
1 change: 0 additions & 1 deletion cli/pkg.generated.mbti
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,3 @@ pub(all) struct CliOptions {
// Type aliases

// Traits

29 changes: 17 additions & 12 deletions cli/render.mbt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
///|
// Renders warnings first and match hits second so parse failures never hide
// Renders warnings first and match hits second so scan diagnostics never hide
// successful findings from other files.
fn render_directory_scan_result(
result : DirectoryScanResult,
Expand All @@ -9,7 +9,7 @@ fn render_directory_scan_result(
if result.scan_trace.length() > 0 {
sections.push(result.scan_trace.join("\n"))
}
let warnings = render_skipped_file_warnings(result.skipped_files)
let warnings = render_scan_warnings(result.warnings)
if warnings != "" {
sections.push(warnings)
}
Expand All @@ -26,20 +26,25 @@ fn render_directory_scan_result(
}

///|
fn render_skipped_file_warnings(skipped_files : Array[SkippedFile]) -> String {
let warnings : Array[String] = []
for skipped in skipped_files {
warnings.push(render_skipped_file_warning(skipped))
fn render_scan_warnings(scan_warnings : Array[ScanWarning]) -> String {
let rendered : Array[String] = []
for warning in scan_warnings {
rendered.push(render_scan_warning(warning))
}
warnings.join("\n")
rendered.join("\n")
}

///|
fn render_skipped_file_warning(skipped : SkippedFile) -> String {
if skipped.block_start_line is Some(line) {
"warning: skipping \{skipped.file} block starting at line \{line}: \{skipped.reason}"
} else {
"warning: skipping \{skipped.file}: \{skipped.reason}"
fn render_scan_warning(warning : ScanWarning) -> String {
match warning {
Parse(parse_warning) =>
if parse_warning.block_start_line is Some(line) {
"warning: skipping \{parse_warning.file} block starting at line \{line}: \{parse_warning.reason}"
} else {
"warning: skipping \{parse_warning.file}: \{parse_warning.reason}"
}
InvalidMoongrepSkipPayload(file, loc) =>
"warning: \{format_location(file, loc)}: #moongrep.skip does not accept a payload; use bare #moongrep.skip"
}
}

Expand Down
153 changes: 103 additions & 50 deletions cli/scan.mbt
Original file line number Diff line number Diff line change
@@ -1,17 +1,23 @@
///|
// Scanner-local result type. The rule model boundary is `rule_model.RuleFinding`;
// parse failures stay here as warnings for the CLI renderer.
priv struct SkippedFile {
// scan diagnostics stay here as warnings for the CLI renderer.
priv struct ParseWarning {
file : String
block_start_line : Int?
reason : String
}

///|
priv enum ScanWarning {
Parse(ParseWarning)
InvalidMoongrepSkipPayload(String, @basic.Location)
}

///|
priv struct DirectoryScanResult {
scan_trace : Array[String]
hits : Array[ScanHit]
skipped_files : Array[SkippedFile]
warnings : Array[ScanWarning]
}

///|
Expand Down Expand Up @@ -42,7 +48,7 @@ priv struct SourceLines {
///|
priv enum ScanEvent {
Trace(String)
Warning(SkippedFile)
Warning(ScanWarning)
Hit(ScanHit)
}

Expand All @@ -53,8 +59,8 @@ priv struct BufferedScanHit {
}

///|
priv struct BufferedSkippedFile {
skipped_file : SkippedFile
priv struct BufferedScanWarning {
warning : ScanWarning
ordinal : Int
}

Expand All @@ -67,7 +73,7 @@ priv struct BufferedSkippedFile {
pub async fn render_scan_command(options : CliOptions) -> String {
let result = collect_directory_hits(options)
if options.output_json {
let warnings = render_skipped_file_warnings(result.skipped_files)
let warnings = render_scan_warnings(result.warnings)
if warnings != "" {
@stdio.stderr.write(warnings + "\n")
}
Expand Down Expand Up @@ -112,29 +118,26 @@ async fn prepare_scan_rules(
async fn collect_directory_hits(options : CliOptions) -> DirectoryScanResult {
let scan_trace : Array[String] = []
let buffered_hits : Array[BufferedScanHit] = []
let buffered_skipped_files : Array[BufferedSkippedFile] = []
let buffered_warnings : Array[BufferedScanWarning] = []
scan_events(options, event => {
match event {
Trace(message) => scan_trace.push(message)
Warning(skipped_file) =>
buffered_skipped_files.push({
skipped_file,
ordinal: buffered_skipped_files.length(),
})
Warning(warning) =>
buffered_warnings.push({ warning, ordinal: buffered_warnings.length() })
Hit(hit) => buffered_hits.push({ hit, ordinal: buffered_hits.length() })
}
})
buffered_hits.sort_by(compare_buffered_scan_hits)
buffered_skipped_files.sort_by(compare_buffered_skipped_files)
buffered_warnings.sort_by(compare_buffered_scan_warnings)
let hits : Array[ScanHit] = []
for buffered in buffered_hits {
hits.push(buffered.hit)
}
let skipped_files : Array[SkippedFile] = []
for buffered in buffered_skipped_files {
skipped_files.push(buffered.skipped_file)
let warnings : Array[ScanWarning] = []
for buffered in buffered_warnings {
warnings.push(buffered.warning)
}
{ scan_trace, hits, skipped_files }
{ scan_trace, hits, warnings }
}

///|
Expand All @@ -151,11 +154,19 @@ fn compare_buffered_scan_hits(
}

///|
fn compare_buffered_skipped_files(
left : BufferedSkippedFile,
right : BufferedSkippedFile,
fn ScanWarning::file(self : ScanWarning) -> String {
match self {
Parse(warning) => warning.file
InvalidMoongrepSkipPayload(file, _) => file
}
}

///|
fn compare_buffered_scan_warnings(
left : BufferedScanWarning,
right : BufferedScanWarning,
) -> Int {
let file_order = left.skipped_file.file.compare(right.skipped_file.file)
let file_order = left.warning.file().compare(right.warning.file())
if file_order == 0 {
left.ordinal.compare(right.ordinal)
} else {
Expand Down Expand Up @@ -346,36 +357,70 @@ fn anonymous_pattern_rule(
}

///|
fn impl_has_moongrep_skip(node : @syntax.Impl) -> Bool {
match node {
TopFuncDef(fun_decl~, ..) => attributes_have_moongrep_skip(fun_decl.attrs)
TopImpl(attrs~, ..) => attributes_have_moongrep_skip(attrs)
TopLetDef(attrs~, ..) => attributes_have_moongrep_skip(attrs)
TopTest(attrs~, ..) => attributes_have_moongrep_skip(attrs)
TopView(attrs~, ..) => attributes_have_moongrep_skip(attrs)
_ => false
priv struct MoongrepSkipAttributes {
skip_structural : Bool
invalid_payload_locations : Array[@basic.Location]
}

///|
fn impl_moongrep_skip_attributes(node : @syntax.Impl) -> MoongrepSkipAttributes {
let attrs = match node {
TopFuncDef(fun_decl~, ..) => fun_decl.attrs
TopImpl(attrs~, ..) => attrs
TopLetDef(attrs~, ..) => attrs
TopTest(attrs~, ..) => attrs
TopView(attrs~, ..) => attrs
_ => return { skip_structural: false, invalid_payload_locations: [] }
}
analyze_moongrep_skip_attributes(attrs)
}

///|
fn attributes_have_moongrep_skip(
fn analyze_moongrep_skip_attributes(
attrs : @list.List[@attribute.Attribute],
) -> Bool {
) -> MoongrepSkipAttributes {
let mut skip_structural = false
let invalid_payload_locations : Array[@basic.Location] = []
for attr in attrs {
if attribute_is_moongrep_skip(attr) {
return true
match attr.parsed {
Some(Ident(id)) if attribute_id_is_moongrep_skip(id) =>
if raw_moongrep_skip_is_bare(attr.raw) {
skip_structural = true
} else {
invalid_payload_locations.push(attr.loc)
}
Some(Apply(id, _)) if attribute_id_is_moongrep_skip(id) =>
invalid_payload_locations.push(attr.loc)
None if unparsed_attribute_targets_moongrep_skip(attr.raw) =>
invalid_payload_locations.push(attr.loc)
_ => ()
}
}
false
{ skip_structural, invalid_payload_locations }
}

///|
fn attribute_id_is_moongrep_skip(id : @attribute.Id) -> Bool {
id.qual == Some("moongrep") && id.name == "skip"
}

///|
fn attribute_is_moongrep_skip(attr : @attribute.Attribute) -> Bool {
match attr.parsed {
Some(Ident(id)) | Some(Apply(id, _)) =>
id.qual == Some("moongrep") && id.name == "skip"
_ => false
fn raw_moongrep_skip_is_bare(raw : String) -> Bool {
let prefix = "#moongrep.skip"
raw.has_prefix(prefix) && raw[prefix.length():].is_blank()
}

///|
fn unparsed_attribute_targets_moongrep_skip(raw : String) -> Bool {
let prefix = "#moongrep.skip"
if !raw.has_prefix(prefix) {
return false
}
if raw.length() == prefix.length() {
return true
}
let next = raw[prefix.length()]
!(next is ('a'..='z' | 'A'..='Z' | '0'..='9' | '_'))
}

///|
Expand Down Expand Up @@ -436,21 +481,29 @@ async fn scan_source_file(
// A malformed matching block does not prevent other matching blocks in
// the same file from being scanned.
receive(
Warning({
file,
block_start_line: if has_multiple_blocks {
Some(block.start_line)
} else {
None
},
reason: "parse failed due to \{compact_scan_reports(reports)}",
}),
Warning(
Parse({
file,
block_start_line: if has_multiple_blocks {
Some(block.start_line)
} else {
None
},
reason: "parse failed due to \{compact_scan_reports(reports)}",
}),
),
)
continue
}
for node in impls {
let untyped_node = @untyped_ast.from_impl(node)
if !impl_has_moongrep_skip(node) {
let skip = impl_moongrep_skip_attributes(node)
for loc in skip.invalid_payload_locations {
receive(
Warning(InvalidMoongrepSkipPayload(file, block.rebase_location(loc))),
)
}
if !skip.skip_structural {
let structural_hits = @rule_apply.apply_structural_scan_plan_to_node(
file, untyped_node, block_plan,
)
Expand Down
Loading
Loading