Skip to content

fix: export extraction misses/phantoms for TS unicode escapes, Erlang arity, GFM tables, NFC/NFD (closes #437, partial) #457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
0xLeif wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: export extraction misses/phantoms for TS unicode escapes, Erlang arity, GFM tables, NFC/NFD (closes #437, partial) #457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 25 additions & 20 deletions src/exports/erlang.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ static ERL_EXPORT_ATTR: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?s)-export\s*\(\s*\[\s*([^]]*)\s*\]\s*\)").unwrap());

/// Function name within export list: name/arity
static ERL_FUN_ARITY: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"'?\b(\w+)'?/\d+").unwrap());
static ERL_FUN_ARITY: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"'?\b(\w+)'?/(\d+)").unwrap());

/// `-compile(export_all).` or `-compile([export_all, ...]).`: a compiler
/// directive that exports every top-level function in the module, regardless
Expand All @@ -39,8 +40,12 @@ pub fn extract_exports(content: &str) -> Vec<String> {
if let Some(list_match) = caps.get(1) {
let list_str = list_match.as_str();
for f_caps in ERL_FUN_ARITY.captures_iter(list_str) {
if let Some(name) = f_caps.get(1) {
let n = name.as_str().to_string();
if let (Some(name), Some(arity)) = (f_caps.get(1), f_caps.get(2)) {
// Erlang export identity is name/arity: `area/1` and
// `area/2` are different exports. Emitting the bare name
// made one of them unmatchable (phantom) and hid arity
// mismatches entirely.
let n = format!("{}/{}", name.as_str(), arity.as_str());
if !symbols.contains(&n) {
symbols.push(n);
}
Expand Down Expand Up @@ -86,10 +91,10 @@ mul(A, B) -> A * B.
helper() -> ok.
"#;
let symbols = extract_exports(src);
assert!(symbols.contains(&"add".to_string()));
assert!(symbols.contains(&"sub".to_string()));
assert!(symbols.contains(&"mul".to_string()));
assert!(symbols.contains(&"DummyClass".to_string()));
assert!(symbols.contains(&"add/2".to_string()));
assert!(symbols.contains(&"sub/2".to_string()));
assert!(symbols.contains(&"mul/2".to_string()));
assert!(symbols.contains(&"DummyClass/0".to_string()));
assert!(!symbols.contains(&"helper".to_string()));
}

Expand All @@ -109,8 +114,8 @@ real_fn(X) -> X.
fake_fn(X) -> X.
"#;
let symbols = extract_exports(src);
assert!(symbols.contains(&"real_fn".to_string()));
assert!(!symbols.contains(&"fake_fn".to_string()));
assert!(symbols.contains(&"real_fn/1".to_string()));
assert!(!symbols.contains(&"fake_fn/1".to_string()));
}

#[test]
Expand Down Expand Up @@ -180,9 +185,9 @@ real_fn2(A, B) -> A + B.
fake_fn(X) -> X.
"#;
let symbols = extract_exports(src);
assert!(symbols.contains(&"real_fn".to_string()));
assert!(symbols.contains(&"real_fn2".to_string()));
assert!(!symbols.contains(&"fake_fn".to_string()));
assert!(symbols.contains(&"real_fn/1".to_string()));
assert!(symbols.contains(&"real_fn2/2".to_string()));
assert!(!symbols.iter().any(|s| s.starts_with("fake_fn/")));
}

#[test]
Expand All @@ -203,8 +208,8 @@ add(A, B) -> A + B.
sub(A, B) -> A - B.
"#;
let symbols = extract_exports(src);
assert!(symbols.contains(&"add".to_string()));
assert!(symbols.contains(&"sub".to_string()));
assert!(symbols.contains(&"add/2".to_string()));
assert!(symbols.contains(&"sub/2".to_string()));
}

#[test]
Expand Down Expand Up @@ -251,11 +256,11 @@ handle_call(_, _, State) -> {reply, ok, State}.
describe_internal() -> ok.
"#;
let symbols = extract_exports(src);
assert!(symbols.contains(&"start_link".to_string()));
assert!(symbols.contains(&"validate".to_string()));
assert!(symbols.contains(&"init".to_string()));
assert!(symbols.contains(&"handle_call".to_string()));
assert!(!symbols.contains(&"describe".to_string()));
assert!(!symbols.contains(&"describe_internal".to_string()));
assert!(symbols.contains(&"start_link/0".to_string()));
assert!(symbols.contains(&"validate/1".to_string()));
assert!(symbols.contains(&"init/1".to_string()));
assert!(symbols.contains(&"handle_call/3".to_string()));
assert!(!symbols.iter().any(|s| s.starts_with("describe/")));
assert!(!symbols.iter().any(|s| s.starts_with("describe_internal/")));
}
}
66 changes: 63 additions & 3 deletions src/exports/typescript.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,58 @@ use tree_sitter::{Parser, Tree};
/// instead of `Name`. `declare` is accepted as an optional modifier so
/// ambient declarations (`export declare function/class/const ...`) are not
/// silently dropped.
/// An identifier token that may contain JS unicode escapes (`\uXXXX`,
/// `\u{...}`) — legal in JS/TS identifiers; truncating at the backslash
/// produces a bogus export name (`caf\u0061` → `caf`).
const JS_IDENT: &str = r"(?:\w|\\u(?:\{[0-9A-Fa-f]{1,6}\}|[0-9A-Fa-f]{4}))+";

static EXPORT_DECL: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"export\s+(?:declare\s+)?(?:async\s+)?(?:abstract\s+)?(?:const\s+enum|function|class|interface|type|const|enum)\s+(\w+)")
.unwrap()
Regex::new(&format!(
r"export\s+(?:declare\s+)?(?:async\s+)?(?:abstract\s+)?(?:const\s+enum|function|class|interface|type|const|enum)\s+({JS_IDENT})"
))
.unwrap()
});

/// Decode `\uXXXX` / `\u{...}` escapes inside a captured identifier.
/// Anything malformed is left as-is (the extractor never invents text).
fn decode_js_identifier_escapes(name: &str) -> String {
if !name.contains('\\') {
return name.to_string();
}
let mut out = String::with_capacity(name.len());
let mut rest = name;
while let Some(pos) = rest.find("\\u") {
out.push_str(&rest[..pos]);
let after = &rest[pos + 2..];
let (digits, consumed) = if let Some(braced) = after.strip_prefix('{') {
match braced.find('}') {
Some(end) => (&braced[..end], end + 2),
None => {
out.push_str("\\u");
rest = after;
continue;
}
}
} else if after.len() >= 4 {
(&after[..4], 4)
} else {
out.push_str("\\u");
rest = after;
continue;
};
match u32::from_str_radix(digits, 16).ok().and_then(char::from_u32) {
Some(ch) => out.push(ch),
None => {
out.push_str("\\u");
out.push_str(&after[..consumed]);
}
}
rest = &after[consumed..];
}
out.push_str(rest);
out
}

/// export type { Name, Name2 }
static RE_EXPORT_TYPE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"export\s+type\s*\{([^}]+)\}").unwrap());
Expand Down Expand Up @@ -88,7 +135,7 @@ pub fn extract_exports_with_resolver(
// Direct exports: export function/class/interface/type/const/enum
for caps in EXPORT_DECL.captures_iter(&stripped) {
if let Some(name) = caps.get(1) {
symbols.push(name.as_str().to_string());
symbols.push(decode_js_identifier_escapes(name.as_str()));
}
}

Expand Down Expand Up @@ -592,6 +639,19 @@ fn strip_comments_preserving_strings(content: &str) -> String {
mod tests {
use super::*;

#[test]
fn test_unicode_escape_identifiers_not_truncated() {
// `caf\u0061` is the legal JS identifier `cafa`; truncating at the
// backslash produced a bogus `caf` export and an unfixable phantom.
let symbols = extract_exports("export const caf\\u0061 = 4;\nexport const b = 2;\n");
assert!(symbols.contains(&"cafa".to_string()), "{symbols:?}");
assert!(!symbols.contains(&"caf".to_string()), "{symbols:?}");
assert!(symbols.contains(&"b".to_string()));
// Brace form.
let symbols = extract_exports("export const \\u{62}eta = 1;\n");
assert!(symbols.contains(&"beta".to_string()), "{symbols:?}");
}

#[test]
fn test_basic_exports() {
let src = r#"
Expand Down
Loading