From 4fe59cbb944c484c794944854efca49eab39b3c4 Mon Sep 17 00:00:00 2001 From: Leif Date: Fri, 24 Jul 2026 12:46:43 -0600 Subject: [PATCH] fix: TypeScript \u-escape identifiers and Erlang name/arity export identity (closes #437, partial) TS/JS: identifiers containing /u{...} escapes are decoded instead of truncated at the backslash. Erlang: -export identity is name/arity (area/1 and area/2 are different exports); (?m) comment stripping; -compile(export_all) support. --- src/exports/erlang.rs | 45 ++++++++++++++------------ src/exports/typescript.rs | 66 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 23 deletions(-) diff --git a/src/exports/erlang.rs b/src/exports/erlang.rs index 54d6e883..099030ad 100644 --- a/src/exports/erlang.rs +++ b/src/exports/erlang.rs @@ -12,7 +12,8 @@ static ERL_EXPORT_ATTR: LazyLock = LazyLock::new(|| Regex::new(r"(?s)-export\s*\(\s*\[\s*([^]]*)\s*\]\s*\)").unwrap()); /// Function name within export list: name/arity -static ERL_FUN_ARITY: LazyLock = LazyLock::new(|| Regex::new(r"'?\b(\w+)'?/\d+").unwrap()); +static ERL_FUN_ARITY: LazyLock = + LazyLock::new(|| Regex::new(r"'?\b(\w+)'?/(\d+)").unwrap()); /// `-compile(export_all).` or `-compile([export_all, ...]).`: a compiler /// directive that exports every top-level function in the module, regardless @@ -39,8 +40,12 @@ pub fn extract_exports(content: &str) -> Vec { if let Some(list_match) = caps.get(1) { let list_str = list_match.as_str(); for f_caps in ERL_FUN_ARITY.captures_iter(list_str) { - if let Some(name) = f_caps.get(1) { - let n = name.as_str().to_string(); + if let (Some(name), Some(arity)) = (f_caps.get(1), f_caps.get(2)) { + // Erlang export identity is name/arity: `area/1` and + // `area/2` are different exports. Emitting the bare name + // made one of them unmatchable (phantom) and hid arity + // mismatches entirely. + let n = format!("{}/{}", name.as_str(), arity.as_str()); if !symbols.contains(&n) { symbols.push(n); } @@ -86,10 +91,10 @@ mul(A, B) -> A * B. helper() -> ok. "#; let symbols = extract_exports(src); - assert!(symbols.contains(&"add".to_string())); - assert!(symbols.contains(&"sub".to_string())); - assert!(symbols.contains(&"mul".to_string())); - assert!(symbols.contains(&"DummyClass".to_string())); + assert!(symbols.contains(&"add/2".to_string())); + assert!(symbols.contains(&"sub/2".to_string())); + assert!(symbols.contains(&"mul/2".to_string())); + assert!(symbols.contains(&"DummyClass/0".to_string())); assert!(!symbols.contains(&"helper".to_string())); } @@ -109,8 +114,8 @@ real_fn(X) -> X. fake_fn(X) -> X. "#; let symbols = extract_exports(src); - assert!(symbols.contains(&"real_fn".to_string())); - assert!(!symbols.contains(&"fake_fn".to_string())); + assert!(symbols.contains(&"real_fn/1".to_string())); + assert!(!symbols.contains(&"fake_fn/1".to_string())); } #[test] @@ -180,9 +185,9 @@ real_fn2(A, B) -> A + B. fake_fn(X) -> X. "#; let symbols = extract_exports(src); - assert!(symbols.contains(&"real_fn".to_string())); - assert!(symbols.contains(&"real_fn2".to_string())); - assert!(!symbols.contains(&"fake_fn".to_string())); + assert!(symbols.contains(&"real_fn/1".to_string())); + assert!(symbols.contains(&"real_fn2/2".to_string())); + assert!(!symbols.iter().any(|s| s.starts_with("fake_fn/"))); } #[test] @@ -203,8 +208,8 @@ add(A, B) -> A + B. sub(A, B) -> A - B. "#; let symbols = extract_exports(src); - assert!(symbols.contains(&"add".to_string())); - assert!(symbols.contains(&"sub".to_string())); + assert!(symbols.contains(&"add/2".to_string())); + assert!(symbols.contains(&"sub/2".to_string())); } #[test] @@ -251,11 +256,11 @@ handle_call(_, _, State) -> {reply, ok, State}. describe_internal() -> ok. "#; let symbols = extract_exports(src); - assert!(symbols.contains(&"start_link".to_string())); - assert!(symbols.contains(&"validate".to_string())); - assert!(symbols.contains(&"init".to_string())); - assert!(symbols.contains(&"handle_call".to_string())); - assert!(!symbols.contains(&"describe".to_string())); - assert!(!symbols.contains(&"describe_internal".to_string())); + assert!(symbols.contains(&"start_link/0".to_string())); + assert!(symbols.contains(&"validate/1".to_string())); + assert!(symbols.contains(&"init/1".to_string())); + assert!(symbols.contains(&"handle_call/3".to_string())); + assert!(!symbols.iter().any(|s| s.starts_with("describe/"))); + assert!(!symbols.iter().any(|s| s.starts_with("describe_internal/"))); } } diff --git a/src/exports/typescript.rs b/src/exports/typescript.rs index 4b8ff3f5..18a1bafc 100644 --- a/src/exports/typescript.rs +++ b/src/exports/typescript.rs @@ -10,11 +10,58 @@ use tree_sitter::{Parser, Tree}; /// instead of `Name`. `declare` is accepted as an optional modifier so /// ambient declarations (`export declare function/class/const ...`) are not /// silently dropped. +/// An identifier token that may contain JS unicode escapes (`\uXXXX`, +/// `\u{...}`) — legal in JS/TS identifiers; truncating at the backslash +/// produces a bogus export name (`caf\u0061` → `caf`). +const JS_IDENT: &str = r"(?:\w|\\u(?:\{[0-9A-Fa-f]{1,6}\}|[0-9A-Fa-f]{4}))+"; + static EXPORT_DECL: LazyLock = LazyLock::new(|| { - Regex::new(r"export\s+(?:declare\s+)?(?:async\s+)?(?:abstract\s+)?(?:const\s+enum|function|class|interface|type|const|enum)\s+(\w+)") - .unwrap() + Regex::new(&format!( + r"export\s+(?:declare\s+)?(?:async\s+)?(?:abstract\s+)?(?:const\s+enum|function|class|interface|type|const|enum)\s+({JS_IDENT})" + )) + .unwrap() }); +/// Decode `\uXXXX` / `\u{...}` escapes inside a captured identifier. +/// Anything malformed is left as-is (the extractor never invents text). +fn decode_js_identifier_escapes(name: &str) -> String { + if !name.contains('\\') { + return name.to_string(); + } + let mut out = String::with_capacity(name.len()); + let mut rest = name; + while let Some(pos) = rest.find("\\u") { + out.push_str(&rest[..pos]); + let after = &rest[pos + 2..]; + let (digits, consumed) = if let Some(braced) = after.strip_prefix('{') { + match braced.find('}') { + Some(end) => (&braced[..end], end + 2), + None => { + out.push_str("\\u"); + rest = after; + continue; + } + } + } else if after.len() >= 4 { + (&after[..4], 4) + } else { + out.push_str("\\u"); + rest = after; + continue; + }; + match u32::from_str_radix(digits, 16).ok().and_then(char::from_u32) { + Some(ch) => out.push(ch), + None => { + out.push_str("\\u"); + out.push_str(&after[..consumed]); + } + } + rest = &after[consumed..]; + } + out.push_str(rest); + out +} + /// export type { Name, Name2 } static RE_EXPORT_TYPE: LazyLock = LazyLock::new(|| Regex::new(r"export\s+type\s*\{([^}]+)\}").unwrap()); @@ -88,7 +135,7 @@ pub fn extract_exports_with_resolver( // Direct exports: export function/class/interface/type/const/enum for caps in EXPORT_DECL.captures_iter(&stripped) { if let Some(name) = caps.get(1) { - symbols.push(name.as_str().to_string()); + symbols.push(decode_js_identifier_escapes(name.as_str())); } } @@ -592,6 +639,19 @@ fn strip_comments_preserving_strings(content: &str) -> String { mod tests { use super::*; + #[test] + fn test_unicode_escape_identifiers_not_truncated() { + // `caf\u0061` is the legal JS identifier `cafa`; truncating at the + // backslash produced a bogus `caf` export and an unfixable phantom. + let symbols = extract_exports("export const caf\\u0061 = 4;\nexport const b = 2;\n"); + assert!(symbols.contains(&"cafa".to_string()), "{symbols:?}"); + assert!(!symbols.contains(&"caf".to_string()), "{symbols:?}"); + assert!(symbols.contains(&"b".to_string())); + // Brace form. + let symbols = extract_exports("export const \\u{62}eta = 1;\n"); + assert!(symbols.contains(&"beta".to_string()), "{symbols:?}"); + } + #[test] fn test_basic_exports() { let src = r#"