From aa01b18ecbedf65144a22695ba2df2f6eedd5012 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:13:23 -0400 Subject: [PATCH 01/29] fix: convert native capture byte offsets to UTF-16 string indexes (C11) Rust Tree-sitter captures expose UTF-8 byte offsets. rangeFromNativeCapture copied them straight into Range, and locals-and-exports.ts published them as string indexes for every query-driven-locals language, corrupting ranges and collapsing cross-file references for any symbol preceded by non-ASCII text. Extract the byte->string index map (already implemented once in ProjectedSyntaxTree) into a shared src/native/byteIndex.ts module reused by both ProjectedSyntaxTree and rangeFromNativeCapture, built once per file and reused across every capture instead of rescanning per offset. Also fixes a compounding defect in Python's named-import regex, which used an ASCII-only character class and silently dropped every import binding for a non-ASCII imported name (PEP 3131 permits Unicode identifiers). --- src/indexer/imports/python.ts | 6 ++- src/indexer/locals-and-exports.ts | 22 ++++++-- src/native/byteIndex.ts | 85 +++++++++++++++++++++++++++++++ src/native/projectedTree.ts | 66 ++---------------------- src/native/queryResults.ts | 18 +++++-- 5 files changed, 125 insertions(+), 72 deletions(-) create mode 100644 src/native/byteIndex.ts diff --git a/src/indexer/imports/python.ts b/src/indexer/imports/python.ts index 2bf71f74..4253455d 100644 --- a/src/indexer/imports/python.ts +++ b/src/indexer/imports/python.ts @@ -116,7 +116,9 @@ export async function collectPythonImportsFromSource(context: PythonImportExtrac await pushStarImport(context, mod); continue; } - const aliasMatch = item.match(/^([A-Za-z_][\w_]*)(?:\s+as\s+([A-Za-z_][\w_]*))?$/); + // PEP 3131 permits Unicode identifiers (XID_Start/XID_Continue); an ASCII-only + // character class here silently drops every non-ASCII imported name's binding. + const aliasMatch = item.match(/^([\p{L}_][\p{L}\p{N}_]*)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?$/u); if (!aliasMatch) continue; const imported = aliasMatch[1]!; const local = aliasMatch[2] ?? imported; @@ -124,7 +126,7 @@ export async function collectPythonImportsFromSource(context: PythonImportExtrac } } - const importPattern = /^(?:\s*)import\s+([A-Za-z_][\w.]*)\s*(?:as\s+([A-Za-z_][\w_]*))?/gm; + const importPattern = /^(?:\s*)import\s+([\p{L}_][\p{L}\p{N}_.]*)\s*(?:as\s+([\p{L}_][\p{L}\p{N}_]*))?/gmu; for (const match of pySrc.matchAll(importPattern)) { const dotted = match[1]!; const local = match[2] ?? dotted.split(".")[0]!; diff --git a/src/indexer/locals-and-exports.ts b/src/indexer/locals-and-exports.ts index e92e5dc3..6ddbce89 100644 --- a/src/indexer/locals-and-exports.ts +++ b/src/indexer/locals-and-exports.ts @@ -1,6 +1,7 @@ import type { LogLevel } from "../logging.js"; import { isGraphOnlyLanguage } from "../documentLinks.js"; import { capturesByName, capturesNamed, rangeFromNativeCapture } from "../native/queryResults.js"; +import { buildByteToStringIndexMap, type ByteToStringIndexMap } from "../native/byteIndex.js"; import { ProjectedSyntaxTree } from "../native/projectedTree.js"; import { assertNativeRequiredAvailable, @@ -378,6 +379,14 @@ export function collectLocalsAndExportsFromSource( return tree; }; + // Lazily build once: converts every native capture's UTF-8 byte offsets to UTF-16 + // string indexes in O(1) per capture instead of rescanning the source per offset. + let byteIndexMap: ByteToStringIndexMap | null = null; + const ensureByteIndexMap = (): ByteToStringIndexMap => { + if (!byteIndexMap) byteIndexMap = buildByteToStringIndexMap(source); + return byteIndexMap; + }; + const locals: SymbolDef[] = []; const seenLocals = new Set(); const toKind = (s: string): SymbolKind => { @@ -431,7 +440,7 @@ export function collectLocalsAndExportsFromSource( for (const match of nativeQueries.locals) { for (const capture of match.captures) { if (capture.name !== "name" && capture.name !== "tname") continue; - const nativeRange = rangeFromNativeCapture(capture); + const nativeRange = rangeFromNativeCapture(capture, ensureByteIndexMap()); const node = enrichmentTree?.rootNode.descendantForIndex(nativeRange.start.index ?? 0, nativeRange.end.index ?? 0) ?? undefined; @@ -528,7 +537,7 @@ export function collectLocalsAndExportsFromSource( ): void => { const nodeForCapture = (capture: NativeCapture | undefined): SyntaxNodeLike | undefined => { if (!capture || !treeForEnrichment) return undefined; - const range = rangeFromNativeCapture(capture); + const range = rangeFromNativeCapture(capture, ensureByteIndexMap()); return treeForEnrichment.rootNode.descendantForIndex(range.start.index ?? 0, range.end.index ?? 0) ?? undefined; }; @@ -749,7 +758,12 @@ export function collectLocalsAndExportsFromSource( if (map["cjs_export_name"] && map["cjs_fn"]) { const exportedAs = map["cjs_export_name"].text; const fnNode = nodeForCapture(map["cjs_fn"]); - const sym = buildSymbolDef(exportedAs, SymbolKind.Function, rangeFromNativeCapture(map["cjs_fn"]), fnNode); + const sym = buildSymbolDef( + exportedAs, + SymbolKind.Function, + rangeFromNativeCapture(map["cjs_fn"], ensureByteIndexMap()), + fnNode, + ); locals.push(sym); exports.push({ type: "local", exportedAs, target: sym }); continue; @@ -792,7 +806,7 @@ export function collectLocalsAndExportsFromSource( const sym = buildSymbolDef( "__default_export__", SymbolKind.Default, - rangeFromNativeCapture(map["anon_default"]), + rangeFromNativeCapture(map["anon_default"], ensureByteIndexMap()), defaultNode, ); locals.push(sym); diff --git a/src/native/byteIndex.ts b/src/native/byteIndex.ts new file mode 100644 index 00000000..0559a1db --- /dev/null +++ b/src/native/byteIndex.ts @@ -0,0 +1,85 @@ +/** + * Tree-sitter native captures expose UTF-8 byte offsets (Rust `start_byte()`/`end_byte()` and a + * byte-relative `Point.column`), while codegraph's `Range` type and JS `String.slice` operate on + * UTF-16 code units. This module builds the byte -> string-index conversion table once per source + * file so every capture in that file converts in O(1) instead of re-scanning the source per offset. + */ + +export type ByteToStringIndexMap = { + readonly isAscii: boolean; + readonly sourceLength: number; + readonly byteToStringIndex: Uint32Array; + readonly lineStartBytes: readonly number[]; +}; + +const EMPTY_BYTE_TO_STRING_INDEX = new Uint32Array(0); +const EMPTY_LINE_START_BYTES: readonly number[] = []; + +export function buildByteToStringIndexMap(source: string): ByteToStringIndexMap { + const byteLength = Buffer.byteLength(source, "utf8"); + if (byteLength === source.length) { + // Pure ASCII: byte offsets and UTF-16 indexes coincide, so skip building the table. + return { + isAscii: true, + sourceLength: source.length, + byteToStringIndex: EMPTY_BYTE_TO_STRING_INDEX, + lineStartBytes: EMPTY_LINE_START_BYTES, + }; + } + + const byteToStringIndex = new Uint32Array(byteLength + 1); + const lineStartBytes: number[] = [0]; + let byteOffset = 0; + let stringIndex = 0; + + while (stringIndex < source.length) { + const codePoint = source.codePointAt(stringIndex); + if (codePoint === undefined) break; + + const charStringLength = codePoint > 0xffff ? 2 : 1; + const charByteLength = utf8ByteLengthForCodePoint(codePoint); + + for (let offset = 1; offset < charByteLength; offset += 1) { + byteToStringIndex[byteOffset + offset] = stringIndex; + } + + byteOffset += charByteLength; + stringIndex += charStringLength; + byteToStringIndex[byteOffset] = stringIndex; + + if (codePoint === 10) { + lineStartBytes.push(byteOffset); + } + } + + byteToStringIndex[byteOffset] = source.length; + return { isAscii: false, sourceLength: source.length, byteToStringIndex, lineStartBytes }; +} + +export function stringIndexForByte(map: ByteToStringIndexMap, byteIndex: number): number { + if (map.isAscii) return Math.max(0, Math.min(byteIndex, map.sourceLength)); + const bounded = Math.max(0, Math.min(byteIndex, map.byteToStringIndex.length - 1)); + return map.byteToStringIndex[bounded] ?? map.sourceLength; +} + +/** + * Converts a Tree-sitter `Point` (0-based row, byte-offset-within-row column) into the + * equivalent 0-based row/column pair expressed in UTF-16 code units. + */ +export function stringPositionForBytePoint( + map: ByteToStringIndexMap, + point: { row: number; column: number }, +): { row: number; column: number } { + if (map.isAscii) return { row: point.row, column: point.column }; + const lineStartByte = map.lineStartBytes[point.row] ?? 0; + const lineStartIndex = stringIndexForByte(map, lineStartByte); + const pointIndex = stringIndexForByte(map, lineStartByte + point.column); + return { row: point.row, column: Math.max(0, pointIndex - lineStartIndex) }; +} + +function utf8ByteLengthForCodePoint(codePoint: number): number { + if (codePoint <= 0x7f) return 1; + if (codePoint <= 0x7ff) return 2; + if (codePoint <= 0xffff) return 3; + return 4; +} diff --git a/src/native/projectedTree.ts b/src/native/projectedTree.ts index fb7bb77e..5e5748f8 100644 --- a/src/native/projectedTree.ts +++ b/src/native/projectedTree.ts @@ -1,4 +1,5 @@ import type { NativePoint, NativeSyntaxNode, NativeSyntaxTree } from "./treeSitterNative.js"; +import { buildByteToStringIndexMap, stringIndexForByte, stringPositionForBytePoint, type ByteToStringIndexMap } from "./byteIndex.js"; export type ProjectedPosition = { row: number; @@ -8,15 +9,12 @@ export type ProjectedPosition = { export class ProjectedSyntaxTree { readonly source: string; private readonly nodesById: Map; - private readonly byteToStringIndex: Uint32Array; - private readonly lineStartBytes: number[]; + private readonly byteMap: ByteToStringIndexMap; readonly rootNode: ProjectedSyntaxNode; constructor(source: string, tree: NativeSyntaxTree) { this.source = source; - const sourceByteMap = buildSourceByteMap(source); - this.byteToStringIndex = sourceByteMap.byteToStringIndex; - this.lineStartBytes = sourceByteMap.lineStartBytes; + this.byteMap = buildByteToStringIndexMap(source); this.nodesById = new Map(); for (const node of tree.nodes) { this.nodesById.set(node.id, new ProjectedSyntaxNode(this, node)); @@ -33,18 +31,11 @@ export class ProjectedSyntaxTree { } stringIndexForByte(byteIndex: number): number { - const bounded = Math.max(0, Math.min(byteIndex, this.byteToStringIndex.length - 1)); - return this.byteToStringIndex[bounded] ?? this.source.length; + return stringIndexForByte(this.byteMap, byteIndex); } positionForPoint(point: NativePoint): ProjectedPosition { - const lineStartByte = this.lineStartBytes[point.row] ?? 0; - const lineStartIndex = this.stringIndexForByte(lineStartByte); - const pointIndex = this.stringIndexForByte(lineStartByte + point.column); - return { - row: point.row, - column: Math.max(0, pointIndex - lineStartIndex), - }; + return stringPositionForBytePoint(this.byteMap, point); } } @@ -156,50 +147,3 @@ function comparePosition(left: ProjectedPosition, right: ProjectedPosition): num return left.column - right.column; } -type SourceByteMap = { - byteToStringIndex: Uint32Array; - lineStartBytes: number[]; -}; - -function buildSourceByteMap(source: string): SourceByteMap { - const byteToStringIndex = new Uint32Array(Buffer.byteLength(source, "utf8") + 1); - const lineStartBytes: number[] = [0]; - let byteOffset = 0; - let stringIndex = 0; - - while (stringIndex < source.length) { - const codePoint = source.codePointAt(stringIndex); - if (codePoint === undefined) break; - - const charStringLength = codePoint > 0xffff ? 2 : 1; - const charByteLength = utf8ByteLengthForCodePoint(codePoint); - - for (let offset = 1; offset < charByteLength; offset += 1) { - byteToStringIndex[byteOffset + offset] = stringIndex; - } - - byteOffset += charByteLength; - stringIndex += charStringLength; - byteToStringIndex[byteOffset] = stringIndex; - - if (codePoint === 10) { - lineStartBytes.push(byteOffset); - } - } - - byteToStringIndex[byteOffset] = source.length; - return { byteToStringIndex, lineStartBytes }; -} - -function utf8ByteLengthForCodePoint(codePoint: number): number { - if (codePoint <= 0x7f) { - return 1; - } - if (codePoint <= 0x7ff) { - return 2; - } - if (codePoint <= 0xffff) { - return 3; - } - return 4; -} diff --git a/src/native/queryResults.ts b/src/native/queryResults.ts index 79beec8c..2e64ccf7 100644 --- a/src/native/queryResults.ts +++ b/src/native/queryResults.ts @@ -1,4 +1,5 @@ import type { Range } from "../types.js"; +import { stringIndexForByte, stringPositionForBytePoint, type ByteToStringIndexMap } from "./byteIndex.js"; import type { NativeCapture, NativeMatch } from "./treeSitterNative.js"; export function capturesByName(match: NativeMatch): Record { @@ -13,17 +14,24 @@ export function capturesNamed(match: NativeMatch, name: string): NativeCapture[] return match.captures.filter((capture) => capture.name === name); } -export function rangeFromNativeCapture(capture: NativeCapture): Range { +/** + * Rust's Tree-sitter captures expose UTF-8 byte offsets. `Range` and every downstream + * consumer (source slicing, portable handles, rename edits) expect UTF-16 string indexes, + * so every capture must convert through the caller's per-file `byteIndexMap` here. + */ +export function rangeFromNativeCapture(capture: NativeCapture, byteIndexMap: ByteToStringIndexMap): Range { + const startPosition = stringPositionForBytePoint(byteIndexMap, capture.start); + const endPosition = stringPositionForBytePoint(byteIndexMap, capture.end); return { start: { line: capture.start.row + 1, - column: capture.start.column + 1, - index: capture.start.index, + column: startPosition.column + 1, + index: stringIndexForByte(byteIndexMap, capture.start.index), }, end: { line: capture.end.row + 1, - column: capture.end.column + 1, - index: capture.end.index, + column: endPosition.column + 1, + index: stringIndexForByte(byteIndexMap, capture.end.index), }, }; } From b5416beae0a05bf0f8f23accc396242a0525a9b4 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:13:34 -0400 Subject: [PATCH 02/29] fix: correct git path encoding and subdirectory blob-hash resolution (C1, C12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - listChangedFiles: use -z + NUL-delimited output instead of newline-split with .trim(), which corrupted every non-ASCII filename (git quotes and octal-escapes them) and stripped legitimate leading/trailing whitespace. - decodeGitPath: reconstruct UTF-8 from octal byte escapes correctly. The previous implementation decoded each \NNN escape as its own code point, mojibaking every multi-byte character ("caf\303\251.ts" -> "café.ts"). - initiateFile: accept quoted a/ and b/ diff --git headers, independently per side, since git quotes each path only if that side needs it. Unquoted headers previously required literal "a/", so a quoted header (any non-ASCII, changed) silently dropped the whole file from the parsed diff. - getGitBlobHashes: pipe absolute paths to 'git hash-object --stdin-paths', which resolves stdin paths against the repository root, not the spawned cwd (unlike ls-files), so a project root that is a repo subdirectory discarded every git signature and fell back to full content hashing. --- src/impact/parse.ts | 103 ++++++++++++++++++++++++++++++++++++-------- src/util/git.ts | 29 ++++++++----- 2 files changed, 104 insertions(+), 28 deletions(-) diff --git a/src/impact/parse.ts b/src/impact/parse.ts index d6d42df8..cfb7aae0 100644 --- a/src/impact/parse.ts +++ b/src/impact/parse.ts @@ -137,19 +137,57 @@ function decodeGitPath(rawPath: string): string { return trimmed; } + // Git quotes a path when it contains non-ASCII or special bytes, escaping each raw byte + // independently as `\NNN` (octal). A multi-byte UTF-8 character becomes several consecutive + // `\NNN` escapes that must be recombined as raw bytes and decoded together as UTF-8 - + // decoding each escape as its own code point (the previous approach) mojibakes every + // non-ASCII path (e.g. "café.ts" became "café.ts"). const inner = trimmed.slice(1, -1); - const decoded = inner.replace(/\\(\\|"|n|r|t|[0-7]{1,3})/g, (match, token: string) => { - if (token === "\\") return "\\"; - if (token === '"') return '"'; - if (token === "n") return "\n"; - if (token === "r") return "\r"; - if (token === "t") return "\t"; - if (/^[0-7]{1,3}$/.test(token)) { - return String.fromCharCode(parseInt(token, 8)); + const bytes: number[] = []; + for (let index = 0; index < inner.length; ) { + const char = inner[index]!; + if (char !== "\\") { + const codePoint = inner.codePointAt(index)!; + bytes.push(...Buffer.from(String.fromCodePoint(codePoint), "utf8")); + index += codePoint > 0xffff ? 2 : 1; + continue; } - return match; - }); - return decoded; + const octal = inner.slice(index + 1, index + 4).match(/^[0-7]{1,3}/); + if (octal) { + bytes.push(parseInt(octal[0], 8) & 0xff); + index += 1 + octal[0].length; + continue; + } + const next = inner[index + 1]; + if (next === "\\" || next === '"') { + bytes.push(next.charCodeAt(0)); + index += 2; + continue; + } + if (next === "n") { + bytes.push(0x0a); + index += 2; + continue; + } + if (next === "r") { + bytes.push(0x0d); + index += 2; + continue; + } + if (next === "t") { + bytes.push(0x09); + index += 2; + continue; + } + // Unrecognized escape: keep the backslash literally. + bytes.push(0x5c); + index += 1; + } + return Buffer.from(bytes).toString("utf8"); +} + +function stripDiffGitPrefix(pathValue: string, prefix: "a/" | "b/"): string { + return pathValue.startsWith(prefix) ? pathValue.slice(prefix.length) : pathValue; } function parseHeaderLine(currentFile: ParsedFileChange, line: string): void { @@ -203,19 +241,50 @@ function parseHeaderLine(currentFile: ParsedFileChange, line: string): void { } } -function initiateFile(line: string): ParsedFileChange | null { - const match = line.match(/^diff --git a\/(.+?) b\/(.+)$/); - if (!match) return null; +const DIFF_GIT_HEADER_PREFIX = "diff --git "; +const QUOTED_PATH_SEGMENT = `"(?:[^"\\\\]|\\\\.)*"`; +// Git quotes each side of the header independently, so a rename between an ASCII and a +// non-ASCII path (or vice versa) can have only one side quoted. Quoted branches are tried +// first since they are unambiguous (the closing quote is exact); the unquoted/unquoted +// fallback keeps the original earliest-" b/"-split heuristic, which is the best a regex can +// do when neither side is delimited (a literal " b/" inside an unquoted path is a +// pre-existing, accepted limitation, unchanged by this fix). +const DIFF_GIT_HEADER_BOTH_QUOTED = new RegExp(`^(${QUOTED_PATH_SEGMENT}) (${QUOTED_PATH_SEGMENT})$`); +const DIFF_GIT_HEADER_A_QUOTED = new RegExp(`^(${QUOTED_PATH_SEGMENT}) b\\/(.+)$`); +const DIFF_GIT_HEADER_B_QUOTED = new RegExp(`^a\\/(.+?) (${QUOTED_PATH_SEGMENT})$`); +const DIFF_GIT_HEADER_PLAIN = /^a\/(.+?) b\/(.+)$/; + +function buildInitiatedFile(aSpec: string, bSpec: string): ParsedFileChange { + const aPath = stripDiffGitPrefix(decodeGitPath(aSpec), "a/"); + const bPath = stripDiffGitPrefix(decodeGitPath(bSpec), "b/"); return { - path: decodeGitPath(match[2]!), + path: bPath, kind: "modified" as const, oldPath: "", hunks: [], - _oldPathFromHeader: decodeGitPath(match[1]!), - _newPathFromHeader: decodeGitPath(match[2]!), + _oldPathFromHeader: aPath, + _newPathFromHeader: bPath, }; } +function initiateFile(line: string): ParsedFileChange | null { + if (!line.startsWith(DIFF_GIT_HEADER_PREFIX)) return null; + const remainder = line.slice(DIFF_GIT_HEADER_PREFIX.length); + + const bothQuoted = remainder.match(DIFF_GIT_HEADER_BOTH_QUOTED); + if (bothQuoted) return buildInitiatedFile(bothQuoted[1]!, bothQuoted[2]!); + + const aQuoted = remainder.match(DIFF_GIT_HEADER_A_QUOTED); + if (aQuoted) return buildInitiatedFile(aQuoted[1]!, `b/${aQuoted[2]}`); + + const bQuoted = remainder.match(DIFF_GIT_HEADER_B_QUOTED); + if (bQuoted) return buildInitiatedFile(`a/${bQuoted[1]}`, bQuoted[2]!); + + const plain = remainder.match(DIFF_GIT_HEADER_PLAIN); + if (!plain) return null; + return buildInitiatedFile(`a/${plain[1]}`, `b/${plain[2]}`); +} + function initiateHunk(line: string): Hunk | null { const match = line.match(/^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@/); if (!match) return null; diff --git a/src/util/git.ts b/src/util/git.ts index cba5fb5f..29809473 100644 --- a/src/util/git.ts +++ b/src/util/git.ts @@ -211,14 +211,17 @@ export function assertSafeRevision(value: string, label: string): string { export function gitDiffArgs(base: string, head: string, extraArgs: string[] = []): string[] { const safeBase = assertSafeRevision(base, "base"); + // Explicit so rename detection stops depending on the user's `diff.renames` config + // (git defaults it to true since 2.9, but a disabled config would silently change output). + const renameArgs = ["--find-renames"]; if (isGitWorktreeSentinel(head)) { - return ["diff", ...extraArgs, "--end-of-options", safeBase]; + return ["diff", ...renameArgs, ...extraArgs, "--end-of-options", safeBase]; } if (isGitIndexSentinel(head)) { - return ["diff", "--cached", ...extraArgs, "--end-of-options", safeBase]; + return ["diff", "--cached", ...renameArgs, ...extraArgs, "--end-of-options", safeBase]; } const safeHead = assertSafeRevision(head, "head"); - return ["diff", ...extraArgs, "--end-of-options", `${safeBase}..${safeHead}`]; + return ["diff", ...renameArgs, ...extraArgs, "--end-of-options", `${safeBase}..${safeHead}`]; } export async function getGitHead(projectRoot: string): Promise { @@ -300,8 +303,11 @@ export async function getGitBlobHashes( .map((line) => line.trim()) .filter((rel) => rel && relFileSet.has(rel)); if (!trackedRel.length) return new Map(); + // hash-object --stdin-paths resolves stdin paths against the repository root, not the + // spawned cwd (unlike ls-files), so projectRoot-relative paths break whenever projectRoot + // is a subdirectory of the repo. Absolute paths resolve correctly regardless of root depth. const { stdout: hashStdout } = await runGit(projectRoot, ["hash-object", "--stdin-paths"], { - input: trackedRel.join("\n"), + input: trackedRel.map((rel) => path.resolve(projectRoot, rel)).join("\n"), }); const hashes = hashStdout .split(/\r?\n/) @@ -354,10 +360,10 @@ export async function listChangedFiles( head?: string | undefined; }, ): Promise { - let args = ["diff", "--name-only", "--diff-filter=ACDMRTUXB"]; + let args = ["diff", "--find-renames", "--name-only", "-z", "--diff-filter=ACDMRTUXB"]; if (opts.base) { const head = opts.head ?? "HEAD"; - args = gitDiffArgs(opts.base, head, ["--name-only", "--diff-filter=ACDMRTUXB"]); + args = gitDiffArgs(opts.base, head, ["--name-only", "-z", "--diff-filter=ACDMRTUXB"]); } else if (opts.changedSince) { args.push("--end-of-options", assertSafeRevision(opts.changedSince, "changedSince")); } else { @@ -366,10 +372,11 @@ export async function listChangedFiles( args.push("--"); try { const stdout = await runGitCollectStdout(projectRoot, args); - const relFiles = stdout - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); + // -z NUL-delimits entries; git also quotes/octal-escapes non-ASCII bytes in the + // unquoted -name-only form, corrupting them, so -z is required, not cosmetic. The + // trailing split segment is always empty, not a filename, and a real filename can + // legitimately start or end with whitespace, so filter without trimming. + const relFiles = stdout.split("\0").filter(Boolean); const out: string[] = []; for (const rel of relFiles) { const abs = normalizePath(path.resolve(projectRoot, rel)); @@ -431,7 +438,7 @@ export async function getUnifiedDiff( head?: string | undefined; }, ): Promise { - let args = ["diff", "--unified=0", "--no-color", "--diff-filter=ACDMRTUXB"]; + let args = ["diff", "--find-renames", "--unified=0", "--no-color", "--diff-filter=ACDMRTUXB"]; if (opts.base) { const head = opts.head ?? "HEAD"; args = gitDiffArgs(opts.base, head, ["--unified=0", "--no-color", "--diff-filter=ACDMRTUXB"]); From 31b08aa5d6377897aefc1890477acfd06fbb939a Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:13:39 -0400 Subject: [PATCH 03/29] test: cover subdirectory-root git blob hash resolution (C1) Regression test: project root nested inside the repo now returns git signatures with no 'Failed to read Git blob hashes' warning. Fails before the fix (hash-object received cwd-relative paths and the whole call returned an empty Map with a warning); passes after. --- tests/cache-invalidation.test.ts | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 51575a2b..7f79a7eb 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -860,6 +860,39 @@ describe("Cache invalidation and strict hashing", () => { } }); + it("resolves git signatures when the project root is a repository subdirectory (C1)", async () => { + const root = await mkTmpDir("dg-git-sig-subdir-root-"); + runGit(root, ["init"]); + runGit(root, ["config", "user.email", "cache@test.local"]); + runGit(root, ["config", "user.name", "Cache Test"]); + + // `git hash-object --stdin-paths` resolves stdin paths against the repository root, not + // the spawned cwd, unlike `git ls-files`. A project root that is a subdirectory of the + // repo previously fed cwd-relative paths straight into that call, so every path failed + // to open and the whole call silently discarded every git signature for the build. + const subdirRoot = path.join(root, "src"); + await fsp.mkdir(subdirRoot, { recursive: true }); + const filePath = path.join(subdirRoot, "a.ts"); + await fsp.writeFile(filePath, "export const a = 1;\n", "utf8"); + runGit(root, ["add", "-A"]); + runGit(root, ["commit", "-m", "init"]); + + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const hashes = await gitModule.getGitBlobHashes(subdirRoot, [filePath]); + + expect(hashes.size).toBe(1); + const hash = hashes.get(normalize(filePath)); + expect(typeof hash).toBe("string"); + expect(hash?.length).toBe(40); + expect(warnSpy.mock.calls.some((call) => String(call[0]).includes("Failed to read Git blob hashes"))).toBe( + false, + ); + } finally { + warnSpy.mockRestore(); + } + }); + it("surfaces a genuine git invocation failure instead of silently discarding signatures", async () => { const root = await mkTmpDir("dg-git-sig-invocation-failure-"); // No `git init`: the directory is not a repository, so `git ls-files` genuinely fails From 85ec36d33e9ddc46c6c162c5ed352a7af2ca5517 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:13:45 -0400 Subject: [PATCH 04/29] fix: include typeOnly in graph edge dedup key (C3) mergeUniqueEdges deduped on from/raw/target only, so a runtime import and a type-only import to the same target (import type { T } from './util'; import { f } from './util';) collapsed onto one edge, silently dropping whichever was processed second. Add typeOnly to the merge key and reuse the existing hasBetterProvenance helper (now exported from graph-edge-collector.ts, matching its use in deduplicateEdges) to keep the stronger-provenance edge when two candidates otherwise share a key. Test fails before the fix (1 edge survives) and passes after (both the typeOnly and runtime edges survive). --- src/graph-builder.ts | 19 +++++++++++-------- src/graph-edge-collector.ts | 2 +- tests/fast-graph-edgecases.test.ts | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/graph-builder.ts b/src/graph-builder.ts index 5638bfa3..93a52359 100644 --- a/src/graph-builder.ts +++ b/src/graph-builder.ts @@ -14,7 +14,7 @@ import type { GraphCacheEntry } from "./graphs/types.js"; import { supportForFile, type LanguageExtensionMap } from "./languages.js"; import type { BuildReport } from "./indexer/types.js"; import type { ParsedFileContext } from "./indexer/parse-context.js"; -import { collectEdgesForFile } from "./graph-edge-collector.js"; +import { collectEdgesForFile, hasBetterProvenance } from "./graph-edge-collector.js"; import { buildSqlFactCache, sqlCorpusSignature } from "./sql/sourceGraph.js"; type GraphFileSignature = { sig: string; gitSig?: string; cacheSig?: string }; @@ -106,17 +106,20 @@ export async function collectGraph( }; const mergeUniqueEdges = (...edgeGroups: Edge[][]): Edge[] => { - const merged: Edge[] = []; - const seen = new Set(); + const byKey = new Map(); for (const group of edgeGroups) { for (const edge of group) { - const key = `${edge.from}::${edge.raw}::${edge.to.type === "file" ? edge.to.path : `external:${edge.to.name}`}`; - if (seen.has(key)) continue; - seen.add(key); - merged.push(edge); + const target = edge.to.type === "file" ? edge.to.path : `external:${edge.to.name}`; + // typeOnly is part of identity: a runtime import and a type-only import to the same + // target are distinct edges (e.g. `import { X }` plus `import type { X }`), and + // collapsing them on from/raw/target alone silently drops the weaker of the two. + const kind = edge.typeOnly ? "type-only" : "runtime"; + const key = `${edge.from}::${edge.raw}::${target}::${kind}`; + const previous = byKey.get(key); + if (!previous || hasBetterProvenance(edge, previous)) byKey.set(key, edge); } } - return merged; + return [...byKey.values()]; }; if (graph.edges.length) { diff --git a/src/graph-edge-collector.ts b/src/graph-edge-collector.ts index 29518597..1d7f89aa 100644 --- a/src/graph-edge-collector.ts +++ b/src/graph-edge-collector.ts @@ -54,7 +54,7 @@ export function deduplicateEdges(edges: Edge[], rawIsIdentity = false): Edge[] { return [...deduplicated.values()]; } -function hasBetterProvenance(candidate: Edge, previous: Edge): boolean { +export function hasBetterProvenance(candidate: Edge, previous: Edge): boolean { let candidateResolutionRank = 0; if (candidate.resolved === "precise") candidateResolutionRank = 2; else if (candidate.resolved === "heuristic") candidateResolutionRank = 1; diff --git a/tests/fast-graph-edgecases.test.ts b/tests/fast-graph-edgecases.test.ts index d2ac3588..a4c9dd16 100644 --- a/tests/fast-graph-edgecases.test.ts +++ b/tests/fast-graph-edgecases.test.ts @@ -26,6 +26,28 @@ describe("Fast graph edge cases", () => { expect(fromMainFast.some((e) => e.typeOnly === true)).toBe(true); }); + it("keeps both a runtime and a type-only edge to the same target (C3)", async () => { + const root = await mkTmpDir("dg-fast-typeonly-both-"); + const util = `export type T = { n: number };\nexport function f(){ return 1 }\n`; + const main = `import type { T } from './util';\nimport { f } from './util';\nconst x: T = { n: f() };\n`; + const utilPath = path.join(root, "util.ts"); + const mainPath = path.join(root, "main.ts"); + await fsp.writeFile(utilPath, util, "utf8"); + await fsp.writeFile(mainPath, main, "utf8"); + const files = [normalizeTestPath(mainPath), normalizeTestPath(utilPath)]; + + const graph = await collectGraph(root, files); + const toUtil = graph.edges + .filter(edgeFrom(mainPath)) + .filter((edge) => edge.to.type === "file" && edge.to.path === normalizeTestPath(utilPath)); + + // A separate runtime import (`{ f }`) and type-only import (`type { T }`) to the same + // target module must both survive dedup, not collapse onto one entry. + expect(toUtil).toHaveLength(2); + expect(toUtil.some((edge) => edge.typeOnly === true)).toBe(true); + expect(toUtil.some((edge) => !edge.typeOnly)).toBe(true); + }); + it("ignores commented-out imports in fast mode", async () => { const root = await mkTmpDir("dg-fast-comments-"); const commented = `// import x from './x'\n/* import y from './y' */\n/*\nimport z from './z'\n*/\n`; From 675999531b6048022762c2de3840936e48c85571 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:13:50 -0400 Subject: [PATCH 05/29] refactor: extract shared import-option builder for primary/embedded sources (C10) src/indexer/build-index.ts duplicated the native/logLevel/languageExtensions/ onFallbackImportExtraction spread across two collectImportsForFile call sites (primary source vs. embedded SFC blocks). A future option added to only one spread would silently not apply to the other. Extract one sharedImportOptions object used by both. No behavior change today (both spreads were already identical); added a regression test asserting the embedded \n`, + "utf8", + ); + + const report: BuildReport = {}; + await buildProjectIndex(root, { cache: "off", logLevel: "silent", report }); + + const fallback = report.graph?.fallbackImportExtraction; + expect(fallback).toBeDefined(); + const widgetEvent = Object.entries(fallback!.files).find(([file]) => file.endsWith("/Widget.vue")); + expect(widgetEvent?.[1]).toEqual({ language: "css", reason: "query-empty" }); + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } + }); +}); From 227df057732ea1937edaf4ed43b529cc93b7a338 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:13:54 -0400 Subject: [PATCH 06/29] fix: give every fallback-extraction reason a human message (D11) createFallbackImportExtractionHandler only built a human sentence for the reduced-mode reason and for languages supporting native regex recovery; every other reason (fast/query-error/query-empty on a language without recovery support, e.g. css) fell through to the bare label plus a dumped event object: "Regex fallback import extraction { language: 'css', reason: 'query-empty' }". Give every reason its own sentence. Test fails before the fix (message === bare label) and passes after. --- src/indexer/build-cache/reports.ts | 8 ++++- ...allback-import-extraction-messages.test.ts | 35 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/fallback-import-extraction-messages.test.ts diff --git a/src/indexer/build-cache/reports.ts b/src/indexer/build-cache/reports.ts index 6aaa7588..0385cc2a 100644 --- a/src/indexer/build-cache/reports.ts +++ b/src/indexer/build-cache/reports.ts @@ -114,11 +114,17 @@ export function createFallbackImportExtractionHandler( event.reason === "fast" || event.reason === "reduced-mode" || supportsReducedModeRegexRecovery(event.language) ? "debug" : "warn"; - let message = "Regex fallback import extraction"; + let message: string; if (event.reason === "reduced-mode") { message = `Native parser unavailable for ${event.language}; using reduced import extraction.`; } else if (supportsReducedModeRegexRecovery(event.language)) { message = `Native import recovery degraded for ${event.language}; using native-owned fallback extraction.`; + } else if (event.reason === "fast") { + message = `Fast mode active for ${event.language}; using regex-based import extraction instead of the native parser.`; + } else if (event.reason === "query-error") { + message = `Native import query failed for ${event.language}; using regex-based fallback extraction.`; + } else { + message = `Native import query returned no results for ${event.language}; using regex-based fallback extraction to recover additional imports.`; } logWithLevel(opts?.logLevel, severity, message, { language: event.language, diff --git a/tests/fallback-import-extraction-messages.test.ts b/tests/fallback-import-extraction-messages.test.ts new file mode 100644 index 00000000..c6ee30d4 --- /dev/null +++ b/tests/fallback-import-extraction-messages.test.ts @@ -0,0 +1,35 @@ +import { describe, it, expect, vi } from "vitest"; +import { createFallbackImportExtractionHandler } from "../src/indexer/build-cache/reports.js"; +import type { FallbackImportExtractionReason } from "../src/graphs/specifiers.js"; + +describe("Fallback import extraction human messages (D11)", () => { + const cases: Array<{ reason: FallbackImportExtractionReason; language: string; expectSubstring: string }> = [ + // CSS has no regex-recovery support baked into the native layer, so these reasons + // previously fell through to the bare label + dumped event object. + { reason: "query-empty", language: "css", expectSubstring: "returned no results" }, + { reason: "query-error", language: "css", expectSubstring: "query failed" }, + { reason: "fast", language: "css", expectSubstring: "Fast mode active" }, + ]; + + it.each(cases)("gives a human sentence for reason=$reason, language=$language", ({ reason, language, expectSubstring }) => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const debugSpy = vi.spyOn(console, "debug").mockImplementation(() => {}); + try { + const handler = createFallbackImportExtractionHandler(undefined, { logLevel: "debug" }); + handler?.({ language, reason, file: "styles.css" }); + + const allCalls = [...warnSpy.mock.calls, ...debugSpy.mock.calls]; + expect(allCalls).toHaveLength(1); + const message = String(allCalls[0]?.[0] ?? ""); + + // Regression guard for the exact bare label observed on stderr (V7): + // "Regex fallback import extraction { language: 'css', reason: 'query-empty' }" + expect(message).not.toBe("Regex fallback import extraction"); + expect(message).toContain(language); + expect(message).toContain(expectSubstring); + } finally { + warnSpy.mockRestore(); + debugSpy.mockRestore(); + } + }); +}); From 5cd6c027519804dfb3f8bef76e54826a0b1c44fe Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:24:40 -0400 Subject: [PATCH 07/29] fix: accept Unicode identifiers in import/alias extractors (C11 sibling audit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grepped every import/alias extractor for the same ASCII-only [A-Za-z_][\w]* character class already found and fixed in src/indexer/imports/python.ts. Found and fixed real siblings across languages whose grammars permit Unicode identifiers: - src/languages/importStatementParsers.ts: Rust (mod/extern crate/use alias), PHP (use-clause alias), Kotlin (import/alias), Java (import), C# (using/alias) -- these are the PRIMARY per-statement parsers. - src/indexer/imports/languageSpecific.ts: Go's normalizeGoImports alias regex; Java/Kotlin text-fallback duplicates of the above. - src/indexer/imports/nativeCaptures.ts: JS/TS CommonJS destructuring require() object-pattern binding names (const { créer } = require(...)) -- provably dropped the whole binding before the fix (imports: []). - src/graphs/specifiers.ts, src/util/specifiers.ts: Python module-specifier fallback extraction; JS/TS combined fast-mode specifier regex's import-equals alias. Two fixes (JS/TS import-equals module spec, Go named-alias) are included for consistency and defense in depth but could not be shown to change observable behavior in this codebase: the module spec is still recovered via a different alternative/mechanism in both cases. Documented inline in the new test file rather than asserted. Test: tests/import-extraction-unicode-identifiers.test.ts, 7 cases each failing before the corresponding fix and passing after. --- src/graphs/specifiers.ts | 5 +- src/indexer/imports/jsTextImports.ts | 13 +- src/indexer/imports/languageSpecific.ts | 9 +- src/indexer/imports/nativeCaptures.ts | 3 +- src/languages/importStatementParsers.ts | 29 ++-- src/util/specifiers.ts | 9 +- ...ort-extraction-unicode-identifiers.test.ts | 125 ++++++++++++++++++ 7 files changed, 168 insertions(+), 25 deletions(-) create mode 100644 tests/import-extraction-unicode-identifiers.test.ts diff --git a/src/graphs/specifiers.ts b/src/graphs/specifiers.ts index 673076a1..41e5cca8 100644 --- a/src/graphs/specifiers.ts +++ b/src/graphs/specifiers.ts @@ -275,12 +275,13 @@ export function collectModuleSpecifiersFromSource( .map((entry) => entry.trim()) .filter(Boolean); for (const spec of list) { - const parsed = spec.match(/^([A-Za-z_][\w.]*)(?:\s+as\s+[A-Za-z_][\w_]*)?$/); + // Python module/package names permit Unicode identifiers (PEP 3131). + const parsed = spec.match(/^([\p{L}_][\p{L}\p{N}_.]*)(?:\s+as\s+[\p{L}_][\p{L}\p{N}_]*)?$/u); if (parsed?.[1]) out.push({ spec: parsed[1] }); } continue; } - const mFrom = /^\s*from\s+(\.*)([A-Za-z_][\w.]*)?\s+import\b/.exec(stmtText); + const mFrom = /^\s*from\s+(\.*)([\p{L}_][\p{L}\p{N}_.]*)?\s+import\b/u.exec(stmtText); if (mFrom) { const dots = mFrom[1] ?? ""; const name = mFrom[2] ?? ""; diff --git a/src/indexer/imports/jsTextImports.ts b/src/indexer/imports/jsTextImports.ts index b2be56f2..8e75a3ba 100644 --- a/src/indexer/imports/jsTextImports.ts +++ b/src/indexer/imports/jsTextImports.ts @@ -23,13 +23,14 @@ function splitNamedImports(namedBlock: string): string[] { } function parseNamedImportSpecifier(spec: string): { imported: string; local: string; typeOnly: boolean } | null { - const typeOnlyMatch = spec.match(/^type\s+([A-Za-z_$][\w$]*)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/); + // JS/TS identifiers permit Unicode ID_Start/ID_Continue plus $/_, not just ASCII. + const typeOnlyMatch = spec.match(/^type\s+([\p{L}_$][\p{L}\p{N}_$]*)(?:\s+as\s+([\p{L}_$][\p{L}\p{N}_$]*))?$/u); if (typeOnlyMatch) { const imported = typeOnlyMatch[1]!; return { imported, local: typeOnlyMatch[2] ?? imported, typeOnly: true }; } - const namedMatch = spec.match(/^([A-Za-z_$][\w$]*)(?:\s+as\s+([A-Za-z_$][\w$]*))?$/); + const namedMatch = spec.match(/^([\p{L}_$][\p{L}\p{N}_$]*)(?:\s+as\s+([\p{L}_$][\p{L}\p{N}_$]*))?$/u); if (!namedMatch) return null; const imported = namedMatch[1]!; return { imported, local: namedMatch[2] ?? imported, typeOnly: false }; @@ -61,7 +62,7 @@ async function collectEsImports( if (!moduleSpecifier) continue; const typeOnly = typeOnlyImport.test(match[0]); const resolved = await context.resolveFrom(moduleSpecifier); - const namespaceMatch = clause.match(/^\*\s+as\s+([A-Za-z_$][\w$]*)$/); + const namespaceMatch = clause.match(/^\*\s+as\s+([\p{L}_$][\p{L}\p{N}_$]*)$/u); if (namespaceMatch) { context.pushBinding({ kind: "namespace", @@ -110,7 +111,7 @@ async function collectCommonJsRequireDeclarations( maskedSource: string, ): Promise { const defaultRequirePattern = - /(?:^|[;{}])\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)/gm; + /(?:^|[;{}])\s*(?:export\s+)?(?:const|let|var)\s+([\p{L}_$][\p{L}\p{N}_$]*)\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)/gmu; for (const match of source.matchAll(defaultRequirePattern)) { if (!matchStartsInCode(maskedSource, match)) continue; const local = match[1]!; @@ -138,7 +139,7 @@ async function collectCommonJsRequireDeclarations( if (!moduleSpecifier) continue; const resolved = await context.resolveFrom(moduleSpecifier); for (const spec of specs) { - const namedMatch = spec.match(/^([A-Za-z_$][\w$]*)(?::\s*([A-Za-z_$][\w$]*))?$/); + const namedMatch = spec.match(/^([\p{L}_$][\p{L}\p{N}_$]*)(?::\s*([\p{L}_$][\p{L}\p{N}_$]*))?$/u); if (!namedMatch) continue; const imported = namedMatch[1]!; const local = namedMatch[2] ?? imported; @@ -160,7 +161,7 @@ async function collectCommonJsImportEquals( maskedSource: string, ): Promise { const importEqualsPattern = - /(?:^|[;{}])\s*import\s+([A-Za-z_$][\w$]*)\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)/gm; + /(?:^|[;{}])\s*import\s+([\p{L}_$][\p{L}\p{N}_$]*)\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)/gmu; for (const match of source.matchAll(importEqualsPattern)) { if (!matchStartsInCode(maskedSource, match)) continue; const local = match[1]!; diff --git a/src/indexer/imports/languageSpecific.ts b/src/indexer/imports/languageSpecific.ts index 4a15ca29..d1ec223b 100644 --- a/src/indexer/imports/languageSpecific.ts +++ b/src/indexer/imports/languageSpecific.ts @@ -37,7 +37,8 @@ function normalizeGoImports(context: LanguageSpecificImportContext): void { return; } const aliasByFrom = new Map(); - const importPattern = /^\s*(?:import\s+)?(?:(?[._A-Za-z][\w]*)\s+)?["'`](?[^"'`]+)["'`]/gm; + // Go identifiers permit Unicode letters (per the Go spec's "letter" production), not just ASCII. + const importPattern = /^\s*(?:import\s+)?(?:(?[._\p{L}][\p{L}\p{N}_]*)\s+)?["'`](?[^"'`]+)["'`]/gmu; for (const match of context.source.matchAll(importPattern)) { const from = match.groups?.from; if (!from) continue; @@ -85,7 +86,8 @@ async function appendJavaTextImports(context: LanguageSpecificImportContext): Pr if (context.languageId !== "java" || context.getBindings().length) { return; } - const importPattern = /^\s*import\s+(static\s+)?([A-Za-z_][\w.]*(?:\.\*)?)\s*;/gm; + // Java identifiers permit Unicode letters, not just ASCII. + const importPattern = /^\s*import\s+(static\s+)?([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)\s*;/gmu; for (const match of context.source.matchAll(importPattern)) { const isStatic = !!match[1]; const rawSpec = match[2]; @@ -121,7 +123,8 @@ async function appendKotlinTextImports(context: LanguageSpecificImportContext): if (context.languageId !== "kotlin" || context.getBindings().length) { return; } - const importPattern = /^\s*import\s+([A-Za-z_][\w.]*(?:\.\*)?)(?:\s+as\s+([A-Za-z_][\w]*))?\s*$/gm; + // Kotlin identifiers permit Unicode letters, not just ASCII. + const importPattern = /^\s*import\s+([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?\s*$/gmu; for (const match of context.source.matchAll(importPattern)) { const rawSpec = match[1]; if (!rawSpec) continue; diff --git a/src/indexer/imports/nativeCaptures.ts b/src/indexer/imports/nativeCaptures.ts index a28de70f..f6ebcf04 100644 --- a/src/indexer/imports/nativeCaptures.ts +++ b/src/indexer/imports/nativeCaptures.ts @@ -29,7 +29,8 @@ function parseObjectPatternBindings(patternText: string): Array<{ imported: stri const out: Array<{ imported: string; local: string }> = []; for (const part of parts) { const withoutDefault = part.replace(/\s*=\s*.+$/, "").trim(); - const match = withoutDefault.match(/^([A-Za-z_$][\w$]*)(?::\s*([A-Za-z_$][\w$]*))?$/); + // JS/TS identifiers permit Unicode ID_Start/ID_Continue plus $/_, not just ASCII. + const match = withoutDefault.match(/^([\p{L}_$][\p{L}\p{N}_$]*)(?::\s*([\p{L}_$][\p{L}\p{N}_$]*))?$/u); if (!match) continue; const imported = match[1]!; const local = match[2] ?? imported; diff --git a/src/languages/importStatementParsers.ts b/src/languages/importStatementParsers.ts index 68e55220..7491da4c 100644 --- a/src/languages/importStatementParsers.ts +++ b/src/languages/importStatementParsers.ts @@ -22,7 +22,8 @@ export type ParsedRustImportStatement = export function parseRustImportStatement(stmtText: string): ParsedRustImportStatement | null { const trimmed = stmtText.trim(); - const modMatch = trimmed.match(/^mod\s+([A-Za-z_][\w]*)\s*;?$/); + // Rust identifiers permit Unicode XID_Start/XID_Continue, not just ASCII. + const modMatch = trimmed.match(/^mod\s+([\p{L}_][\p{L}\p{N}_]*)\s*;?$/u); if (modMatch?.[1]) { return { kind: "module", @@ -32,7 +33,9 @@ export function parseRustImportStatement(stmtText: string): ParsedRustImportStat }; } - const externMatch = trimmed.match(/^extern\s+crate\s+([A-Za-z_][\w]*)(?:\s+as\s+([A-Za-z_][\w]*))?\s*;?$/); + const externMatch = trimmed.match( + /^extern\s+crate\s+([\p{L}_][\p{L}\p{N}_]*)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?\s*;?$/u, + ); if (externMatch?.[1]) { return { kind: "module", @@ -47,7 +50,7 @@ export function parseRustImportStatement(stmtText: string): ParsedRustImportStat if (!useBody) return null; if (useBody.includes("{") || useBody.includes(",")) return null; - const aliasMatch = useBody.match(/^(.*?)\s+as\s+([A-Za-z_][\w]*)$/); + const aliasMatch = useBody.match(/^(.*?)\s+as\s+([\p{L}_][\p{L}\p{N}_]*)$/u); const rawPath = aliasMatch?.[1]?.trim() ?? useBody; const alias = aliasMatch?.[2]; @@ -141,7 +144,8 @@ function parsePhpImportClause(rawClause: string, importType: PhpImportType): Par memberType = "const"; } const body = (typedMemberMatch?.[2] ?? member).trim(); - const aliasMatch = body.match(/^(.*?)\s+as\s+([A-Za-z_][\w]*)$/i); + // PHP identifiers permit any byte >= 0x80, which in practice means non-ASCII UTF-8. + const aliasMatch = body.match(/^(.*?)\s+as\s+([\p{L}_][\p{L}\p{N}_]*)$/iu); const fullPath = `${prefix}${(aliasMatch?.[1] ?? body).trim()}`; const parts = fullPath.split("\\").filter(Boolean); const imported = parts[parts.length - 1]; @@ -158,7 +162,7 @@ function parsePhpImportClause(rawClause: string, importType: PhpImportType): Par return results; } - const aliasMatch = clause.match(/^(.*?)\s+as\s+([A-Za-z_][\w]*)$/i); + const aliasMatch = clause.match(/^(.*?)\s+as\s+([\p{L}_][\p{L}\p{N}_]*)$/iu); const fullPath = (aliasMatch?.[1] ?? clause).trim(); const parts = fullPath.split("\\").filter(Boolean); const imported = parts[parts.length - 1]; @@ -362,7 +366,10 @@ export type ParsedKotlinImportStatement = }; export function parseKotlinImportStatement(stmtText: string): ParsedKotlinImportStatement | null { - const match = stmtText.trim().match(/^\s*import\s+([A-Za-z_][\w.]*(?:\.\*)?)(?:\s+as\s+([A-Za-z_][\w]*))?\s*$/m); + // Kotlin identifiers permit Unicode letters, not just ASCII. + const match = stmtText + .trim() + .match(/^\s*import\s+([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?\s*$/mu); const rawSpec = match?.[1]; if (!rawSpec) return null; if (rawSpec.endsWith(".*")) { @@ -397,7 +404,8 @@ export type ParsedJavaImportStatement = }; export function parseJavaImportStatement(stmtText: string): ParsedJavaImportStatement | null { - const match = stmtText.trim().match(/^\s*import\s+(static\s+)?([A-Za-z_][\w.]*(?:\.\*)?)\s*;?\s*$/); + // Java identifiers permit Unicode letters, not just ASCII. + const match = stmtText.trim().match(/^\s*import\s+(static\s+)?([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)\s*;?\s*$/u); const rawSpec = match?.[2]; if (!rawSpec) return null; const isStatic = !!match?.[1]; @@ -423,7 +431,8 @@ export function parseJavaImportStatement(stmtText: string): ParsedJavaImportStat export function parseCsharpUsingDirective(stmtText: string): ParsedCsharpUsingDirective | null { const trimmed = stmtText.trim(); - const aliasMatch = trimmed.match(/^(?:global\s+)?using\s+([A-Za-z_][\w]*)\s*=\s*([A-Za-z_][\w.]*)\s*;?$/); + // C# identifiers permit Unicode letter categories, not just ASCII. + const aliasMatch = trimmed.match(/^(?:global\s+)?using\s+([\p{L}_][\p{L}\p{N}_]*)\s*=\s*([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u); if (aliasMatch?.[1] && aliasMatch[2]) { return { from: aliasMatch[2], @@ -432,7 +441,7 @@ export function parseCsharpUsingDirective(stmtText: string): ParsedCsharpUsingDi }; } - const staticMatch = trimmed.match(/^(?:global\s+)?using\s+static\s+([A-Za-z_][\w.]*)\s*;?$/); + const staticMatch = trimmed.match(/^(?:global\s+)?using\s+static\s+([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u); if (staticMatch?.[1]) { return { from: staticMatch[1], @@ -440,7 +449,7 @@ export function parseCsharpUsingDirective(stmtText: string): ParsedCsharpUsingDi }; } - const plainMatch = trimmed.match(/^(?:global\s+)?using\s+([A-Za-z_][\w.]*)\s*;?$/); + const plainMatch = trimmed.match(/^(?:global\s+)?using\s+([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u); if (!plainMatch?.[1]) return null; return { from: plainMatch[1], diff --git a/src/util/specifiers.ts b/src/util/specifiers.ts index 302e1496..c23cf568 100644 --- a/src/util/specifiers.ts +++ b/src/util/specifiers.ts @@ -45,8 +45,10 @@ export function extractJsTsSpecifiers(source: string): ModuleSpecifier[] { const literalMask = buildJsLikeLiteralMask(src); // Capture groups: 1 import-from, 2 side-effect import, 3 export-from, // 4 destructured require, 5 require(), 6 import(), 7 import = require, 8 declare module. + // JS/TS identifiers permit Unicode ID_Start/ID_Continue plus $/_, not just ASCII, so the + // import-equals alias uses \p{L}\p{N} rather than \w (which is ASCII-only). const combined = - /^\s*import\s+[^\n;]*?\s+from\s+["']([^"']+)["']|^\s*import\s+["']([^"']+)["']|\bexport\s+[^\n;]*?\s+from\s+["']([^"']+)["']|\b(?:const|let|var)\s*\{[^}]*\}\s*=\s*require\s*\(\s*["']([^"']+)["']\s*\)|(? { + it("Rust: extern crate alias, use alias, and module name", () => { + expect(parseRustImportStatement("mod créer;")).toEqual({ + kind: "module", + from: "créer", + local: "créer", + isExternCrate: false, + }); + expect(parseRustImportStatement("extern crate créer as créé;")).toEqual({ + kind: "module", + from: "créer", + local: "créé", + isExternCrate: true, + }); + expect(parseRustImportStatement("use std::foo as créer;")).toEqual({ + kind: "member", + from: "std", + imported: "foo", + local: "créer", + }); + }); + + it("PHP: use-clause alias", () => { + expect(parsePhpImportStatement("use App\\Foo as créer;")).toEqual([ + { + kind: "named", + from: "App\\Foo", + imported: "Foo", + local: "créer", + importType: "class", + }, + ]); + }); + + it("Kotlin: import alias", () => { + expect(parseKotlinImportStatement("import com.example.Foo as créer")).toEqual({ + kind: "named", + from: "com.example.Foo", + imported: "Foo", + local: "créer", + }); + }); + + it("Java: import of a Unicode-named class", () => { + expect(parseJavaImportStatement("import com.example.Créer;")).toEqual({ + kind: "named", + from: "com.example.Créer", + imported: "Créer", + isStatic: false, + }); + }); + + it("C#: using alias to a Unicode-named alias", () => { + expect(parseCsharpUsingDirective("using créer = Some.Namespace;")).toEqual({ + from: "Some.Namespace", + alias: "créer", + isStatic: false, + }); + }); + it("Python fallback module-specifier extraction: import/from with Unicode module names", () => { + expect(extractPythonSpecifiers("import créer\n")).toEqual(["créer"]); + expect(extractPythonSpecifiers("from créer import x\n")).toContain("créer"); + }); + + it("JS CommonJS destructuring require(): Unicode property name binding", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "cg-cjs-unicode-destructure-")); + try { + await fsp.writeFile(path.join(root, "dep.js"), "module.exports = { créer() { return 1; } };\n", "utf8"); + await fsp.writeFile(path.join(root, "main.js"), "const { créer } = require('./dep');\ncréer();\n", "utf8"); + + const index = await buildProjectIndex(root, { cache: "off" }); + const mainFile = [...index.byFile.keys()].find((file) => file.endsWith("/main.js"))!; + const mainModule = index.byFile.get(mainFile)!; + // `const { créer } = require('./dep')` is parsed via an object-pattern text regex + // (native captures only expose the whole pattern's text, not per-property names). + // Before the fix the ASCII-only character class matched nothing in "créer" and the + // whole binding was silently dropped -- imports was empty even though the require() + // call and file-level graph edge were both still detected by a separate mechanism. + expect(mainModule.imports).toEqual([ + expect.objectContaining({ kind: "named", local: "créer", imported: "créer", from: "./dep" }), + ]); + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } + }); +}); + +// Two additional sibling fixes were made for the same defect class but could not be proven +// to change observable behavior against this codebase's actual extraction pipeline, so they +// are documented here rather than asserted as regression tests: +// +// - src/util/specifiers.ts's combined JS/TS regex (import X = require(...) alternative): the +// module specifier is still recovered via the separate bare `require(...)` alternative in +// the same combined pattern even when the "import X =" alias fails to match, and +// extractJsTsSpecifiers's return type never exposes the alias/local name at all. The fix +// (Unicode-aware alias class) is still correct and removes a latent dependency on that +// alternative-pattern fallback, but there is no independently observable before/after +// difference through this function's public contract. +// - src/indexer/imports/languageSpecific.ts's normalizeGoImports alias regex: Go's native +// query captures already expose the import alias identifier as raw capture text (unaffected +// by any JS regex), so a named Unicode alias is already correct via the primary native path +// before this fix. The only alias values normalizeGoImports's regex result changes behavior +// for are the single-character "." (dot-import) and "_" (blank-import) sentinels, both of +// which were already covered by the old ASCII character class. The fix is still correct +// (defense in depth against a change to the native query), but not independently provable +// as a behavior change in this codebase today. From e5e7228eb4e0b59da15fd38d6fb1d190c4465bbd Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:27:49 -0400 Subject: [PATCH 08/29] test: cover non-ASCII/space filenames and quoted renames through the git provider (C12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit listChangedFiles/getUnifiedDiff + parseUnifiedDiff now correctly round-trip non-ASCII, space, and leading-space filenames, and a rename to a non-ASCII (quoted-header) path reports kind: 'renamed' with the correct oldPath. Both cases fail before the fix (café.ts corrupted into '"251.ts"' via octal-escape bytes colliding with path.resolve on Windows; the renamed file resolves to the same mangled '"...-renamed.ts"' name) and pass after. --- tests/git-diff-semantics.test.ts | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/git-diff-semantics.test.ts b/tests/git-diff-semantics.test.ts index b38cfef1..fa01696e 100644 --- a/tests/git-diff-semantics.test.ts +++ b/tests/git-diff-semantics.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { listChangedFiles, listUntrackedFiles, getUnifiedDiff } from "../src/util.js"; +import { parseUnifiedDiff } from "../src/impact/parse.js"; import { runGit as git } from "./helpers/git.js"; function makeGitTempDir(prefix: string): Promise { @@ -153,6 +154,67 @@ describe("git diff semantics", () => { }); }); +describe("git diff semantics: non-ASCII, space, and rename path handling (C12)", () => { + it("returns non-ASCII, space, and leading/trailing-space filenames as real UTF-8, not git's quoted/escaped form", async () => { + const root = await makeGitTempDir("codegraph-git-c12-names-"); + try { + git(root, ["init"]); + git(root, ["config", "user.email", "tests@example.com"]); + git(root, ["config", "user.name", "Tests"]); + + await fs.writeFile(path.join(root, "café.ts"), "export const a = 1;\n", "utf8"); + await fs.writeFile(path.join(root, "with space.ts"), "export const b = 1;\n", "utf8"); + await fs.writeFile(path.join(root, " leading.ts"), "export const c = 1;\n", "utf8"); + git(root, ["add", "."]); + git(root, ["commit", "-m", "base"]); + + await fs.writeFile(path.join(root, "café.ts"), "export const a = 2;\n", "utf8"); + await fs.writeFile(path.join(root, "with space.ts"), "export const b = 2;\n", "utf8"); + await fs.writeFile(path.join(root, " leading.ts"), "export const c = 2;\n", "utf8"); + + const changed = await listChangedFiles(root, { changedSince: "HEAD" }); + const names = changed.map((entry) => path.basename(entry)).sort(); + expect(names).toEqual(["café.ts", " leading.ts", "with space.ts"].sort()); + + // getUnifiedDiff returns git's raw output verbatim (still quoted/octal-escaped for + // café.ts, since that quoting comes from git itself); parseUnifiedDiff is what decodes + // it, so assert against the parsed result rather than the raw diff text. + const diff = await getUnifiedDiff(root, { changedSince: "HEAD" }); + const parsedDiff = parseUnifiedDiff(diff); + expect(parsedDiff.files.map((file) => file.path).sort()).toEqual(["café.ts", " leading.ts", "with space.ts"].sort()); + } finally { + await removeGitTempDir(root); + } + }); + + it("propagates a rename to a non-ASCII (quoted) path through listChangedFiles and the parsed diff", async () => { + const root = await makeGitTempDir("codegraph-git-c12-rename-"); + try { + git(root, ["init"]); + git(root, ["config", "user.email", "tests@example.com"]); + git(root, ["config", "user.name", "Tests"]); + + await fs.writeFile(path.join(root, "plain.ts"), "export function run() {\n return 1;\n}\n", "utf8"); + git(root, ["add", "."]); + git(root, ["commit", "-m", "base"]); + + git(root, ["mv", "plain.ts", "café-renamed.ts"]); + git(root, ["add", "-A"]); + + const changed = await listChangedFiles(root, { base: "HEAD", head: "STAGED" }); + expect(changed.map((entry) => path.basename(entry))).toEqual(["café-renamed.ts"]); + + const diff = await getUnifiedDiff(root, { base: "HEAD", head: "STAGED" }); + const parsed = parseUnifiedDiff(diff); + expect(parsed.files).toEqual([ + expect.objectContaining({ kind: "renamed", path: "café-renamed.ts", oldPath: "plain.ts" }), + ]); + } finally { + await removeGitTempDir(root); + } + }); +}); + describe("listUntrackedFiles", () => { it("lists new files Git has not been told to track", async () => { const root = await makeGitTempDir("codegraph-git-untracked-"); From 2a65895b02cceda2fd5c4481dfb23981d82ab41e Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:28:19 -0400 Subject: [PATCH 09/29] test: cover non-ASCII filenames and renames through the review/impact getDiff path (C12) Before the fix, both a modified non-ASCII file and a rename to a non-ASCII path were silently dropped from the parsed diff entirely (diff.files was empty), since initiateFile's regex required a literal unquoted 'a/' prefix and git quotes the header for any non-ASCII path. --- tests/impact-git-provider.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/impact-git-provider.test.ts b/tests/impact-git-provider.test.ts index 6e0c5f82..b295a4ec 100644 --- a/tests/impact-git-provider.test.ts +++ b/tests/impact-git-provider.test.ts @@ -123,6 +123,24 @@ export function extra() { expect(stagedDiff.files.map((file) => file.path).sort()).toEqual(expectedFiles); expect(indexDiff.files.map((file) => file.path).sort()).toEqual(expectedFiles); }); + + it("resolves non-ASCII filenames and a non-ASCII rename through the review/impact diff path (C12)", async () => { + const root = createGitRepo(); + writeFile(root, "café.ts", "export const a = 1;\n"); + writeFile(root, "plain.ts", "export function run() {\n return 1;\n}\n"); + const base = commitAll(root, "initial"); + + writeFile(root, "café.ts", "export const a = 2;\n"); + git(root, ["mv", "plain.ts", "日本-renamed.ts"]); + const head = commitAll(root, "unicode changes"); + + const diff = await getDiff({ provider: "git", cwd: root, base, head }); + + expect(diff.files.map((file) => file.path).sort()).toEqual(["café.ts", "日本-renamed.ts"]); + const renamed = diff.files.find((file) => file.path === "日本-renamed.ts"); + expect(renamed?.kind).toBe("renamed"); + expect(renamed?.oldPath).toBe("plain.ts"); + }); it("rejects when the Git process cannot be spawned", async () => { const root = createGitRepo(); const missingCwd = path.join(root, "missing"); From 7152b12d6c98e4019e95e51a6d490646b5777461 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:29:04 -0400 Subject: [PATCH 10/29] test: cover quoted diff --git headers directly in the unified-diff parser (C12) Unit-level coverage for decodeGitPath/initiateFile: non-ASCII path quoted on both sides, a rename where only the destination needs quoting, and a literal double-quote character in a filename (a synthetic case -- Windows cannot create a file with " in its name, so this exercises the escape decoder directly against hand-written diff text rather than a real repo). All three fail before the fix (parsed.files is empty -- the whole file entry is dropped) and pass after. --- tests/streaming-parser.test.ts | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/streaming-parser.test.ts b/tests/streaming-parser.test.ts index 2884775d..d57267e1 100644 --- a/tests/streaming-parser.test.ts +++ b/tests/streaming-parser.test.ts @@ -151,3 +151,48 @@ ${hunkLines.join("\n")} await expect(parseUnifiedDiffStreaming(stream)).rejects.toThrow("stream error"); }); }); + +describe("Quoted diff --git headers (C12)", () => { + it("decodes a non-ASCII path quoted and octal-escaped on both sides", () => { + // Real `git diff` output for a non-ASCII filename: git quotes the path and escapes each + // UTF-8 byte independently as \\NNN (octal). Recombining those bytes correctly requires + // decoding them as raw bytes and re-parsing as UTF-8, not as individual code points. + const diffText = `diff --git "a/caf\\303\\251.ts" "b/caf\\303\\251.ts" +index 0000000..1111111 100644 +--- "a/caf\\303\\251.ts" ++++ "b/caf\\303\\251.ts" +@@ -1 +1 @@ +-old ++new +`; + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: "café.ts", kind: "modified" })]); + }); + + it("decodes a rename where only the destination side needs quoting", () => { + const diffText = `diff --git a/plain.ts "b/\\346\\227\\245\\346\\234\\254/renamed.ts" +similarity index 100% +rename from plain.ts +rename to "\\346\\227\\245\\346\\234\\254/renamed.ts" +`; + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([ + expect.objectContaining({ path: "日本/renamed.ts", oldPath: "plain.ts", kind: "renamed" }), + ]); + }); + + it("decodes an escaped double-quote character inside a quoted filename", () => { + // A literal `"` in a path is itself one of the characters git must quote/escape; this + // exercises the \\" escape specifically, independent of any octal-byte decoding. + const diffText = `diff --git "a/quote\\"test.ts" "b/quote\\"test.ts" +index 0000000..1111111 100644 +--- "a/quote\\"test.ts" ++++ "b/quote\\"test.ts" +@@ -1 +1 @@ +-old ++new +`; + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: 'quote"test.ts', kind: "modified" })]); + }); +}); From f15343b841fb4850751c79952d4aed23fe46d00a Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:29:50 -0400 Subject: [PATCH 11/29] test: cover rename detection with diff.renames=false configured locally (C4) A repo/user with diff.renames=false previously changed codegraph's own output (a rename reported as an unrelated delete+add pair with no oldPath, since git respects that config by default). gitDiffArgs now always passes --find-renames explicitly. Fails before the fix (kind: 'added'/'deleted' pair, no rename linkage) and passes after (kind: 'renamed', oldPath populated). --- tests/git-diff-semantics.test.ts | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/git-diff-semantics.test.ts b/tests/git-diff-semantics.test.ts index fa01696e..8bd8d6c9 100644 --- a/tests/git-diff-semantics.test.ts +++ b/tests/git-diff-semantics.test.ts @@ -215,6 +215,41 @@ describe("git diff semantics: non-ASCII, space, and rename path handling (C12)", }); }); +describe("git diff semantics: rename detection is deterministic regardless of user config (C4)", () => { + it("still reports a pure rename with diff.renames=false configured locally", async () => { + const root = await makeGitTempDir("codegraph-git-c4-renames-config-"); + try { + git(root, ["init"]); + git(root, ["config", "user.email", "tests@example.com"]); + git(root, ["config", "user.name", "Tests"]); + // A user (or repo) can disable git's default rename detection entirely. gitDiffArgs + // must pass --find-renames explicitly so codegraph's own output does not silently + // depend on this config. + git(root, ["config", "diff.renames", "false"]); + + const original = "export function widget() {\n return 1;\n}\n".repeat(3); + await fs.writeFile(path.join(root, "widget.ts"), original, "utf8"); + git(root, ["add", "."]); + git(root, ["commit", "-m", "base"]); + const base = git(root, ["rev-parse", "HEAD"]); + + git(root, ["mv", "widget.ts", "renamed-widget.ts"]); + git(root, ["add", "-A"]); + git(root, ["commit", "-m", "rename"]); + const head = git(root, ["rev-parse", "HEAD"]); + + const diff = await getUnifiedDiff(root, { base, head }); + const parsed = parseUnifiedDiff(diff); + + expect(parsed.files).toEqual([ + expect.objectContaining({ kind: "renamed", path: "renamed-widget.ts", oldPath: "widget.ts" }), + ]); + } finally { + await removeGitTempDir(root); + } + }); +}); + describe("listUntrackedFiles", () => { it("lists new files Git has not been told to track", async () => { const root = await makeGitTempDir("codegraph-git-untracked-"); From cb867b46aa411962ffcddfbe52767afd93e205bf Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:34:56 -0400 Subject: [PATCH 12/29] test: cover C11 range identity for every usesQueryDrivenLocals language Adds a per-language regression case to tests/languages/{go,java,csharp, kotlin,php,rust,swift,cpp,zig}.test.ts asserting a native-query-driven symbol's range.start.index equals source.indexOf(name) and that slicing the range recovers the identifier text, for a declaration preceded by multibyte text both on an earlier line and on its own line. Eight languages (go, java, csharp, kotlin, php, rust, swift, cpp) use a genuinely Unicode-named declaration, mirroring the Python V1 repro exactly. Zig's grammar is ASCII-only for plain identifiers (needs @"..." for anything else), so its case uses an ASCII declaration name preceded by Unicode comment/string content instead, and says so in the test name. Shared helper: tests/languages/unicodeSymbolRange.ts. All nine fail before the C11 fix (range.start.index off by the byte-length delta of the preceding multibyte text) and pass after. --- tests/languages/cpp.test.ts | 11 +++++++ tests/languages/csharp.test.ts | 12 ++++++++ tests/languages/go.test.ts | 12 ++++++++ tests/languages/java.test.ts | 13 +++++++++ tests/languages/kotlin.test.ts | 12 ++++++++ tests/languages/php.test.ts | 11 +++++++ tests/languages/rust.test.ts | 11 +++++++ tests/languages/swift.test.ts | 12 ++++++++ tests/languages/unicodeSymbolRange.ts | 42 +++++++++++++++++++++++++++ tests/languages/zig.test.ts | 15 ++++++++++ 10 files changed, 151 insertions(+) create mode 100644 tests/languages/unicodeSymbolRange.ts diff --git a/tests/languages/cpp.test.ts b/tests/languages/cpp.test.ts index e98022f1..018019d2 100644 --- a/tests/languages/cpp.test.ts +++ b/tests/languages/cpp.test.ts @@ -7,6 +7,7 @@ import { listCandidateTestFiles } from "../../src/impact/context.js"; import { normalizePath } from "../../src/util/paths.js"; import { runLanguageTests } from "./runner.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; import { C_SUPPORT, CPP_SUPPORT, supportForFile } from "../../src/languages.js"; import { parseSyntaxTree } from "@lzehrung/codegraph-native"; @@ -223,3 +224,13 @@ describe("C++ configured include roots", () => { } }); }); + +describe("C++ Unicode symbol ranges (C11)", () => { + it("publishes a UTF-16 string index for a function name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "widget.cpp", + source: "// café ☕ prüfung\n/* über */ int créer() {\n\treturn 1;\n}\n", + symbolName: "créer", + }); + }); +}); diff --git a/tests/languages/csharp.test.ts b/tests/languages/csharp.test.ts index 93aa0dbc..59fe3bf5 100644 --- a/tests/languages/csharp.test.ts +++ b/tests/languages/csharp.test.ts @@ -4,6 +4,7 @@ import { runLanguageTests } from "./runner.js"; import { createTestIndexFromFiles } from "../test-utils.js"; import { fileIdentityKey } from "../../src/util/paths.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; const definition: LanguageTestDefinition = { id: "csharp", @@ -109,6 +110,17 @@ const definition: LanguageTestDefinition = { runLanguageTests(definition); +describe("C# Unicode symbol ranges (C11)", () => { + it("publishes a UTF-16 string index for a method name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "Widget.cs", + source: + "// café ☕ prüfung\n/* über */ public class Widget {\n\tpublic int Créer() {\n\t\treturn 1;\n\t}\n}\n", + symbolName: "Créer", + }); + }); +}); + describe("C# global using directives", () => { it("keeps global, alias, and static forms as resolved import bindings", async () => { const sampleDir = path.resolve(process.cwd(), "tests", "samples", "csharp"); diff --git a/tests/languages/go.test.ts b/tests/languages/go.test.ts index 1f2e04a2..b15db62f 100644 --- a/tests/languages/go.test.ts +++ b/tests/languages/go.test.ts @@ -1,5 +1,7 @@ +import { describe, it } from "vitest"; import { runLanguageTests } from "./runner.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; const definition: LanguageTestDefinition = { id: "go", @@ -246,3 +248,13 @@ const definition: LanguageTestDefinition = { }; runLanguageTests(definition); + +describe("Go Unicode symbol ranges (C11)", () => { + it("publishes a UTF-16 string index for a function name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "widget.go", + source: "package widget\n\n// café ☕ prüfung\n/* über */ func créer() int {\n\treturn 1\n}\n", + symbolName: "créer", + }); + }); +}); diff --git a/tests/languages/java.test.ts b/tests/languages/java.test.ts index 44c42081..3c98b165 100644 --- a/tests/languages/java.test.ts +++ b/tests/languages/java.test.ts @@ -1,5 +1,7 @@ +import { describe, it } from "vitest"; import { runLanguageTests } from "./runner.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; const definition: LanguageTestDefinition = { id: "java", @@ -221,3 +223,14 @@ const definition: LanguageTestDefinition = { }; runLanguageTests(definition); + +describe("Java Unicode symbol ranges (C11)", () => { + it("publishes a UTF-16 string index for a method name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "Widget.java", + source: + "// café ☕ prüfung\n/* über */ public class Widget {\n\tpublic int créer() {\n\t\treturn 1;\n\t}\n}\n", + symbolName: "créer", + }); + }); +}); diff --git a/tests/languages/kotlin.test.ts b/tests/languages/kotlin.test.ts index cbd7a341..3295f559 100644 --- a/tests/languages/kotlin.test.ts +++ b/tests/languages/kotlin.test.ts @@ -1,5 +1,7 @@ +import { describe, it } from "vitest"; import { runLanguageTests } from "./runner.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; const definition: LanguageTestDefinition = { id: "kotlin", @@ -134,3 +136,13 @@ const definition: LanguageTestDefinition = { }; runLanguageTests(definition); + +describe("Kotlin Unicode symbol ranges (C11)", () => { + it("publishes a UTF-16 string index for a function name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "widget.kt", + source: "// café ☕ prüfung\n/* über */ fun créer(): Int {\n\treturn 1\n}\n", + symbolName: "créer", + }); + }); +}); diff --git a/tests/languages/php.test.ts b/tests/languages/php.test.ts index 1518553a..178922ee 100644 --- a/tests/languages/php.test.ts +++ b/tests/languages/php.test.ts @@ -8,6 +8,7 @@ import { findImplementations } from "../../src/indexer/type-hierarchy.js"; import { createTestIndexFromFiles } from "../test-utils.js"; import { runLanguageTests } from "./runner.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; const definition: LanguageTestDefinition = { id: "php", @@ -666,3 +667,13 @@ class Example { } }); }); + +describe("PHP Unicode symbol ranges (C11)", () => { + it("publishes a UTF-16 string index for a function name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "widget.php", + source: " { + it("publishes a UTF-16 string index for a function name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "widget.rs", + source: "// café ☕ prüfung\n/* über */ fn créer() -> i32 {\n\t1\n}\n", + symbolName: "créer", + }); + }); +}); diff --git a/tests/languages/swift.test.ts b/tests/languages/swift.test.ts index 07cc17ff..02f3b34f 100644 --- a/tests/languages/swift.test.ts +++ b/tests/languages/swift.test.ts @@ -1,5 +1,7 @@ +import { describe, it } from "vitest"; import { runLanguageTests } from "./runner.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; const definition: LanguageTestDefinition = { id: "swift", @@ -95,3 +97,13 @@ const definition: LanguageTestDefinition = { }; runLanguageTests(definition); + +describe("Swift Unicode symbol ranges (C11)", () => { + it("publishes a UTF-16 string index for a function name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "widget.swift", + source: "// café ☕ prüfung\n/* über */ func créer() -> Int {\n\treturn 1\n}\n", + symbolName: "créer", + }); + }); +}); diff --git a/tests/languages/unicodeSymbolRange.ts b/tests/languages/unicodeSymbolRange.ts new file mode 100644 index 00000000..a63fe396 --- /dev/null +++ b/tests/languages/unicodeSymbolRange.ts @@ -0,0 +1,42 @@ +import { expect } from "vitest"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { collectLocalsAndExportsFromSource, parseFile } from "../../src/indexer.js"; + +/** + * C11 regression helper: asserts a native-query-driven symbol's published range is a UTF-16 + * string index identical to `source.indexOf(symbolName)`, and that slicing the range recovers + * the identifier text. The fixture source is expected to carry multibyte (non-ASCII) text both + * on an earlier line and immediately before the declaration on its own line, so both the + * cross-line byte->string line-start offset and the same-line byte->string column offset are + * exercised; before the fix, native capture byte offsets were published unconverted. + */ +export async function expectUnicodeSymbolRangeIdentity(opts: { + fileName: string; + source: string; + symbolName: string; +}): Promise { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "cg-unicode-range-")); + try { + const file = path.join(root, opts.fileName); + await fsp.writeFile(file, opts.source, "utf8"); + const parsed = await parseFile(file); + const mod = collectLocalsAndExportsFromSource(file, parsed.source, parsed.sup, parsed.lang, [], { + tree: parsed.tree, + nativeQueries: parsed.nativeQueries, + }); + const sym = mod.locals.find((s) => s.localName === opts.symbolName); + expect(sym, `expected a local symbol named "${opts.symbolName}" in locals: ${mod.locals.map((l) => l.localName).join(", ")}`).toBeDefined(); + + const expectedIndex = opts.source.indexOf(opts.symbolName); + expect(expectedIndex).toBeGreaterThanOrEqual(0); + expect(sym!.range.start.index, "range.start.index must equal source.indexOf(name)").toBe(expectedIndex); + expect( + opts.source.slice(sym!.range.start.index, sym!.range.start.index + opts.symbolName.length), + "slicing the published range must recover the identifier text", + ).toBe(opts.symbolName); + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } +} diff --git a/tests/languages/zig.test.ts b/tests/languages/zig.test.ts index 9abcdc6e..07108b83 100644 --- a/tests/languages/zig.test.ts +++ b/tests/languages/zig.test.ts @@ -1,5 +1,7 @@ +import { describe, it } from "vitest"; import { runLanguageTests } from "./runner.js"; import type { LanguageTestDefinition } from "./types.js"; +import { expectUnicodeSymbolRangeIdentity } from "./unicodeSymbolRange.js"; const definition: LanguageTestDefinition = { id: "zig", @@ -97,3 +99,16 @@ const definition: LanguageTestDefinition = { }; runLanguageTests(definition); + +describe("Zig symbol ranges after preceding multibyte text (C11)", () => { + // Zig identifiers are ASCII-only (an arbitrary identifier needs @"..." syntax), so this + // uses an ASCII declaration name preceded by multibyte text on an earlier line and on the + // same line, unlike the other languages' Unicode-identifier fixtures. + it("publishes a UTF-16 string index for an ASCII function name preceded by multibyte text", async () => { + await expectUnicodeSymbolRangeIdentity({ + fileName: "widget.zig", + source: '// café ☕ prüfung\nconst greeting = "über"; fn create_widget() i32 {\n return 1;\n}\n', + symbolName: "create_widget", + }); + }); +}); From ea70899f4e0697e5e78c7839fe5de7933bfa0295 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:41:02 -0400 Subject: [PATCH 13/29] test: extend shared goto/references/native-semantic-parity coverage for C11 (per AGENTS.md) AGENTS.md requires cross-file language scenarios to extend tests/goto.test.ts, tests/references.test.ts, and tests/native-semantic-parity.test.ts alongside the per-language file. Adds: - goto.test.ts: resolves a call to a Unicode-named Python function to the exact definition range.start.index (was byte-offset-corrupted, and the binding itself was previously dropped by the ASCII-only import regex, making the whole lookup return not_found before both fixes). - references.test.ts: the exact V1 repro (19 references for a Unicode identifier vs 19 for its ASCII control) as a committed, exact-count regression test, not just the hand-run probe. - native-semantic-parity.test.ts: a new .regressions/unicode_{def,consumer}.py fixture pair added at the END of the shared cases array (to avoid shifting every subsequent snapshot's numeric index) verifying native/reduced-mode parity for Unicode-named symbols. Note: this harness's stable snapshot normalizes to file+line only (not column/index), so it does not independently prove the byte-offset regression the way the goto.test.ts and references.test.ts cases do -- it is still valid required coverage per AGENTS.md, documented here for accuracy. --- .../native-semantic-parity.test.ts.snap | 18 +++++++++ tests/goto.test.ts | 27 +++++++++++++ tests/native-semantic-parity.test.ts | 7 ++++ tests/references.test.ts | 40 +++++++++++++++++++ .../python/.regressions/unicode_consumer.py | 3 ++ .../python/.regressions/unicode_def.py | 3 ++ 6 files changed, 98 insertions(+) create mode 100644 tests/samples/python/.regressions/unicode_consumer.py create mode 100644 tests/samples/python/.regressions/unicode_def.py diff --git a/tests/__snapshots__/native-semantic-parity.test.ts.snap b/tests/__snapshots__/native-semantic-parity.test.ts.snap index 2744d429..d8aab461 100644 --- a/tests/__snapshots__/native-semantic-parity.test.ts.snap +++ b/tests/__snapshots__/native-semantic-parity.test.ts.snap @@ -792,3 +792,21 @@ exports[`native semantic coverage > keeps native semantics stable for representa }, } `; + +exports[`native semantic coverage > keeps native semantics stable for representative language fixtures 48`] = ` +{ + "goto": { + "file": ".regressions/unicode_def.py", + "line": 2, + "status": "ok", + }, + "references": { + "refs": [ + ".regressions/unicode_consumer.py:1", + ".regressions/unicode_consumer.py:3", + ".regressions/unicode_def.py:2", + ], + "status": "ok", + }, +} +`; diff --git a/tests/goto.test.ts b/tests/goto.test.ts index 034d4bbd..452e7837 100644 --- a/tests/goto.test.ts +++ b/tests/goto.test.ts @@ -1669,3 +1669,30 @@ describe("Go to Definition", () => { }); }); }); + +describe("Go to Definition: Unicode identifiers (C11)", () => { + it("resolves a call to a Unicode-named function to its exact identifier position", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "cg-goto-unicode-")); + try { + const defFile = path.join(root, "u1.py").replace(/\\/g, "/"); + const useFile = path.join(root, "consumer.py").replace(/\\/g, "/"); + const defSource = 'x = "ééé"\ndef créer():\n return 1\n'; + const useSource = "from u1 import créer\n\ncréer()\n"; + await fsp.writeFile(defFile, defSource, "utf8"); + await fsp.writeFile(useFile, useSource, "utf8"); + const index = await createTestIndexFromFiles(root, [defFile, useFile]); + + const callColumn = useSource.split("\n")[2]!.indexOf("créer") + 1; + const result = await goToDefinition(index, { file: useFile, line: 3, column: callColumn }); + + expect(result.status).toBe("ok"); + if (result.status !== "ok") return; + expect(fileIdentityKey(result.definition.file)).toBe(fileIdentityKey(defFile)); + // The definition range must land exactly on "créer" in def source, not offset by the + // byte length of the preceding non-ASCII string literal. + expect(result.definition.range.start.index).toBe(defSource.indexOf("créer")); + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/native-semantic-parity.test.ts b/tests/native-semantic-parity.test.ts index 0627eba8..e187008e 100644 --- a/tests/native-semantic-parity.test.ts +++ b/tests/native-semantic-parity.test.ts @@ -806,6 +806,13 @@ nativeDescribe("native semantic coverage", () => { { file: ".regressions/macros.rs", line: 6, column: 5, expectedStatus: "ok" }, { file: ".regressions/macros.rs", line: 1, column: 14, expectedStatus: "ok" }, ), + sampleExpectation( + "python", + [".regressions/unicode_def.py", ".regressions/unicode_consumer.py"], + [{ file: ".regressions/unicode_def.py", names: ["x", "créer"] }], + { file: ".regressions/unicode_consumer.py", line: 3, column: 1, expectedStatus: "ok" }, + { file: ".regressions/unicode_def.py", line: 2, column: 5, expectedStatus: "ok" }, + ), ]; for (const testCase of cases) { diff --git a/tests/references.test.ts b/tests/references.test.ts index f72122d6..62224190 100644 --- a/tests/references.test.ts +++ b/tests/references.test.ts @@ -2014,3 +2014,43 @@ describe("Find References", () => { }); }); }); + +describe("Find References: Unicode identifiers (C11)", () => { + it("finds every cross-file reference to a Unicode-named function, matching an ASCII control", async () => { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), "cg-references-unicode-")); + try { + // Mirrors the audit's V1 repro: a definition file whose source begins with non-ASCII + // text (byte offset drift), consumed twice each by six files. An ASCII control with the + // identical structure proves the counts are byte-offset-driven, not incidental. + const uDefFile = path.join(root, "u1.py").replace(/\\/g, "/"); + await fsp.writeFile(uDefFile, 'x = "ééé"\ndef créer():\n return 1\n', "utf8"); + const uConsumerFiles: string[] = []; + for (let i = 1; i <= 6; i += 1) { + const file = path.join(root, `cu${i}.py`).replace(/\\/g, "/"); + await fsp.writeFile(file, `from u1 import créer\n\ndef use${i}():\n créer()\n créer()\n`, "utf8"); + uConsumerFiles.push(file); + } + + const aDefFile = path.join(root, "a1.py").replace(/\\/g, "/"); + await fsp.writeFile(aDefFile, 'x = "eee"\ndef creer():\n return 1\n', "utf8"); + const aConsumerFiles: string[] = []; + for (let i = 1; i <= 6; i += 1) { + const file = path.join(root, `ca${i}.py`).replace(/\\/g, "/"); + await fsp.writeFile(file, `from a1 import creer\n\ndef use${i}():\n creer()\n creer()\n`, "utf8"); + aConsumerFiles.push(file); + } + + const index = await createTestIndexFromFiles(root, [uDefFile, aDefFile, ...uConsumerFiles, ...aConsumerFiles]); + + const uResult = await testFindReferences(index, uDefFile, 2, "def créer".indexOf("créer") + 1, 19); + expect(uResult.status).toBe("ok"); + if (uResult.status === "ok") expect(uResult.references).toHaveLength(19); + + const aResult = await testFindReferences(index, aDefFile, 2, "def creer".indexOf("creer") + 1, 19); + expect(aResult.status).toBe("ok"); + if (aResult.status === "ok") expect(aResult.references).toHaveLength(19); + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/samples/python/.regressions/unicode_consumer.py b/tests/samples/python/.regressions/unicode_consumer.py new file mode 100644 index 00000000..1be6845a --- /dev/null +++ b/tests/samples/python/.regressions/unicode_consumer.py @@ -0,0 +1,3 @@ +from .unicode_def import créer + +créer() diff --git a/tests/samples/python/.regressions/unicode_def.py b/tests/samples/python/.regressions/unicode_def.py new file mode 100644 index 00000000..af8bc07d --- /dev/null +++ b/tests/samples/python/.regressions/unicode_def.py @@ -0,0 +1,3 @@ +x = "ééé" +def créer(): + return 1 From 9df51f6a8c90f6dcacb05dcb93106f4cb98dc4e7 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:42:06 -0400 Subject: [PATCH 14/29] test: update gitDiffArgs exact-array assertions for --find-renames (C4) Pre-existing test asserted the exact args array; update it to include the --find-renames flag added by the C4 fix, in the same position for each of the worktree/index/range sentinel branches. --- tests/git-revision-safety.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/git-revision-safety.test.ts b/tests/git-revision-safety.test.ts index 73dd1b46..04064eda 100644 --- a/tests/git-revision-safety.test.ts +++ b/tests/git-revision-safety.test.ts @@ -11,9 +11,10 @@ describe("git revision safety", () => { }); it("places --end-of-options immediately before revision arguments in gitDiffArgs", () => { - expect(gitDiffArgs("main", "HEAD")).toEqual(["diff", "--end-of-options", "main..HEAD"]); + expect(gitDiffArgs("main", "HEAD")).toEqual(["diff", "--find-renames", "--end-of-options", "main..HEAD"]); expect(gitDiffArgs("main", "WORKTREE", ["--name-only"])).toEqual([ "diff", + "--find-renames", "--name-only", "--end-of-options", "main", @@ -21,6 +22,7 @@ describe("git revision safety", () => { expect(gitDiffArgs("main", "STAGED", ["--name-only"])).toEqual([ "diff", "--cached", + "--find-renames", "--name-only", "--end-of-options", "main", From a94f7780765e4db19297868ae51cfe08be8597c9 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 01:44:23 -0400 Subject: [PATCH 15/29] style: apply prettier to changed files --- src/languages/importStatementParsers.ts | 4 +- src/native/projectedTree.ts | 8 +++- tests/cache-invalidation.test.ts | 4 +- ...allback-import-extraction-messages.test.ts | 41 ++++++++++--------- tests/git-diff-semantics.test.ts | 4 +- tests/languages/csharp.test.ts | 3 +- tests/languages/java.test.ts | 3 +- tests/languages/unicodeSymbolRange.ts | 5 ++- 8 files changed, 41 insertions(+), 31 deletions(-) diff --git a/src/languages/importStatementParsers.ts b/src/languages/importStatementParsers.ts index 7491da4c..43e84d80 100644 --- a/src/languages/importStatementParsers.ts +++ b/src/languages/importStatementParsers.ts @@ -432,7 +432,9 @@ export function parseCsharpUsingDirective(stmtText: string): ParsedCsharpUsingDi const trimmed = stmtText.trim(); // C# identifiers permit Unicode letter categories, not just ASCII. - const aliasMatch = trimmed.match(/^(?:global\s+)?using\s+([\p{L}_][\p{L}\p{N}_]*)\s*=\s*([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u); + const aliasMatch = trimmed.match( + /^(?:global\s+)?using\s+([\p{L}_][\p{L}\p{N}_]*)\s*=\s*([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u, + ); if (aliasMatch?.[1] && aliasMatch[2]) { return { from: aliasMatch[2], diff --git a/src/native/projectedTree.ts b/src/native/projectedTree.ts index 5e5748f8..a26cdf5a 100644 --- a/src/native/projectedTree.ts +++ b/src/native/projectedTree.ts @@ -1,5 +1,10 @@ import type { NativePoint, NativeSyntaxNode, NativeSyntaxTree } from "./treeSitterNative.js"; -import { buildByteToStringIndexMap, stringIndexForByte, stringPositionForBytePoint, type ByteToStringIndexMap } from "./byteIndex.js"; +import { + buildByteToStringIndexMap, + stringIndexForByte, + stringPositionForBytePoint, + type ByteToStringIndexMap, +} from "./byteIndex.js"; export type ProjectedPosition = { row: number; @@ -146,4 +151,3 @@ function comparePosition(left: ProjectedPosition, right: ProjectedPosition): num } return left.column - right.column; } - diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 7f79a7eb..22b66720 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -885,9 +885,7 @@ describe("Cache invalidation and strict hashing", () => { const hash = hashes.get(normalize(filePath)); expect(typeof hash).toBe("string"); expect(hash?.length).toBe(40); - expect(warnSpy.mock.calls.some((call) => String(call[0]).includes("Failed to read Git blob hashes"))).toBe( - false, - ); + expect(warnSpy.mock.calls.some((call) => String(call[0]).includes("Failed to read Git blob hashes"))).toBe(false); } finally { warnSpy.mockRestore(); } diff --git a/tests/fallback-import-extraction-messages.test.ts b/tests/fallback-import-extraction-messages.test.ts index c6ee30d4..6974501a 100644 --- a/tests/fallback-import-extraction-messages.test.ts +++ b/tests/fallback-import-extraction-messages.test.ts @@ -11,25 +11,28 @@ describe("Fallback import extraction human messages (D11)", () => { { reason: "fast", language: "css", expectSubstring: "Fast mode active" }, ]; - it.each(cases)("gives a human sentence for reason=$reason, language=$language", ({ reason, language, expectSubstring }) => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - const debugSpy = vi.spyOn(console, "debug").mockImplementation(() => {}); - try { - const handler = createFallbackImportExtractionHandler(undefined, { logLevel: "debug" }); - handler?.({ language, reason, file: "styles.css" }); + it.each(cases)( + "gives a human sentence for reason=$reason, language=$language", + ({ reason, language, expectSubstring }) => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const debugSpy = vi.spyOn(console, "debug").mockImplementation(() => {}); + try { + const handler = createFallbackImportExtractionHandler(undefined, { logLevel: "debug" }); + handler?.({ language, reason, file: "styles.css" }); - const allCalls = [...warnSpy.mock.calls, ...debugSpy.mock.calls]; - expect(allCalls).toHaveLength(1); - const message = String(allCalls[0]?.[0] ?? ""); + const allCalls = [...warnSpy.mock.calls, ...debugSpy.mock.calls]; + expect(allCalls).toHaveLength(1); + const message = String(allCalls[0]?.[0] ?? ""); - // Regression guard for the exact bare label observed on stderr (V7): - // "Regex fallback import extraction { language: 'css', reason: 'query-empty' }" - expect(message).not.toBe("Regex fallback import extraction"); - expect(message).toContain(language); - expect(message).toContain(expectSubstring); - } finally { - warnSpy.mockRestore(); - debugSpy.mockRestore(); - } - }); + // Regression guard for the exact bare label observed on stderr (V7): + // "Regex fallback import extraction { language: 'css', reason: 'query-empty' }" + expect(message).not.toBe("Regex fallback import extraction"); + expect(message).toContain(language); + expect(message).toContain(expectSubstring); + } finally { + warnSpy.mockRestore(); + debugSpy.mockRestore(); + } + }, + ); }); diff --git a/tests/git-diff-semantics.test.ts b/tests/git-diff-semantics.test.ts index 8bd8d6c9..c453e039 100644 --- a/tests/git-diff-semantics.test.ts +++ b/tests/git-diff-semantics.test.ts @@ -181,7 +181,9 @@ describe("git diff semantics: non-ASCII, space, and rename path handling (C12)", // it, so assert against the parsed result rather than the raw diff text. const diff = await getUnifiedDiff(root, { changedSince: "HEAD" }); const parsedDiff = parseUnifiedDiff(diff); - expect(parsedDiff.files.map((file) => file.path).sort()).toEqual(["café.ts", " leading.ts", "with space.ts"].sort()); + expect(parsedDiff.files.map((file) => file.path).sort()).toEqual( + ["café.ts", " leading.ts", "with space.ts"].sort(), + ); } finally { await removeGitTempDir(root); } diff --git a/tests/languages/csharp.test.ts b/tests/languages/csharp.test.ts index 59fe3bf5..899384e1 100644 --- a/tests/languages/csharp.test.ts +++ b/tests/languages/csharp.test.ts @@ -114,8 +114,7 @@ describe("C# Unicode symbol ranges (C11)", () => { it("publishes a UTF-16 string index for a method name preceded by multibyte text", async () => { await expectUnicodeSymbolRangeIdentity({ fileName: "Widget.cs", - source: - "// café ☕ prüfung\n/* über */ public class Widget {\n\tpublic int Créer() {\n\t\treturn 1;\n\t}\n}\n", + source: "// café ☕ prüfung\n/* über */ public class Widget {\n\tpublic int Créer() {\n\t\treturn 1;\n\t}\n}\n", symbolName: "Créer", }); }); diff --git a/tests/languages/java.test.ts b/tests/languages/java.test.ts index 3c98b165..99121d1a 100644 --- a/tests/languages/java.test.ts +++ b/tests/languages/java.test.ts @@ -228,8 +228,7 @@ describe("Java Unicode symbol ranges (C11)", () => { it("publishes a UTF-16 string index for a method name preceded by multibyte text", async () => { await expectUnicodeSymbolRangeIdentity({ fileName: "Widget.java", - source: - "// café ☕ prüfung\n/* über */ public class Widget {\n\tpublic int créer() {\n\t\treturn 1;\n\t}\n}\n", + source: "// café ☕ prüfung\n/* über */ public class Widget {\n\tpublic int créer() {\n\t\treturn 1;\n\t}\n}\n", symbolName: "créer", }); }); diff --git a/tests/languages/unicodeSymbolRange.ts b/tests/languages/unicodeSymbolRange.ts index a63fe396..84b85611 100644 --- a/tests/languages/unicodeSymbolRange.ts +++ b/tests/languages/unicodeSymbolRange.ts @@ -27,7 +27,10 @@ export async function expectUnicodeSymbolRangeIdentity(opts: { nativeQueries: parsed.nativeQueries, }); const sym = mod.locals.find((s) => s.localName === opts.symbolName); - expect(sym, `expected a local symbol named "${opts.symbolName}" in locals: ${mod.locals.map((l) => l.localName).join(", ")}`).toBeDefined(); + expect( + sym, + `expected a local symbol named "${opts.symbolName}" in locals: ${mod.locals.map((l) => l.localName).join(", ")}`, + ).toBeDefined(); const expectedIndex = opts.source.indexOf(opts.symbolName); expect(expectedIndex).toBeGreaterThanOrEqual(0); From c1fa3479303f4cfc46bfc8734027cecc2f1db5e3 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sat, 15 Aug 2026 22:48:30 -0400 Subject: [PATCH 16/29] fix: preserve encoded git paths in repairs --- src/impact/parse.ts | 7 ++-- src/util/git.ts | 54 +++++++++++++++++++++--------- tests/cache-invalidation.test.ts | 21 ++++++++++++ tests/native-query-results.test.ts | 34 +++++++++++++++++++ tests/streaming-parser.test.ts | 17 ++++++++++ 5 files changed, 114 insertions(+), 19 deletions(-) create mode 100644 tests/native-query-results.test.ts diff --git a/src/impact/parse.ts b/src/impact/parse.ts index cfb7aae0..68b5ddee 100644 --- a/src/impact/parse.ts +++ b/src/impact/parse.ts @@ -132,9 +132,8 @@ function decodeStreamChunk(decoder: StringDecoder, chunk: unknown): string { } function decodeGitPath(rawPath: string): string { - const trimmed = rawPath.trim(); - if (!trimmed.startsWith('"') || !trimmed.endsWith('"')) { - return trimmed; + if (!rawPath.startsWith('"') || !rawPath.endsWith('"')) { + return rawPath; } // Git quotes a path when it contains non-ASCII or special bytes, escaping each raw byte @@ -142,7 +141,7 @@ function decodeGitPath(rawPath: string): string { // `\NNN` escapes that must be recombined as raw bytes and decoded together as UTF-8 - // decoding each escape as its own code point (the previous approach) mojibakes every // non-ASCII path (e.g. "café.ts" became "café.ts"). - const inner = trimmed.slice(1, -1); + const inner = rawPath.slice(1, -1); const bytes: number[] = []; for (let index = 0; index < inner.length; ) { const char = inner[index]!; diff --git a/src/util/git.ts b/src/util/git.ts index 29809473..6a4d9562 100644 --- a/src/util/git.ts +++ b/src/util/git.ts @@ -12,6 +12,7 @@ import { logWithLevel, type LogLevel } from "../logging.js"; export const DEFAULT_GIT_TIMEOUT_MS = 30_000; const gitRepositoryChecks = new Map>(); +const MAX_GIT_HASH_OBJECT_ARGUMENT_BYTES = 24 * 1024; let gitExecutableForTests: string | null = null; @@ -297,22 +298,9 @@ export async function getGitBlobHashes( const { stdout: trackedStdout } = await runGit(projectRoot, ["ls-files", "-z"], { maxBuffer: 64 * 1024 * 1024, }); - const trackedRel = trackedStdout - .toString() - .split("\0") - .map((line) => line.trim()) - .filter((rel) => rel && relFileSet.has(rel)); + const trackedRel = trackedStdout.split("\0").filter((rel) => rel && relFileSet.has(rel)); if (!trackedRel.length) return new Map(); - // hash-object --stdin-paths resolves stdin paths against the repository root, not the - // spawned cwd (unlike ls-files), so projectRoot-relative paths break whenever projectRoot - // is a subdirectory of the repo. Absolute paths resolve correctly regardless of root depth. - const { stdout: hashStdout } = await runGit(projectRoot, ["hash-object", "--stdin-paths"], { - input: trackedRel.map((rel) => path.resolve(projectRoot, rel)).join("\n"), - }); - const hashes = hashStdout - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); + const hashes = await hashGitPaths(projectRoot, trackedRel); if (hashes.length !== trackedRel.length) { logWithLevel( opts?.logLevel, @@ -346,6 +334,42 @@ export async function getGitBlobHashes( } } +async function hashGitPaths(projectRoot: string, trackedRel: string[]): Promise { + const batches: string[][] = []; + let currentBatch: string[] = []; + let currentBatchBytes = 0; + + for (const rel of trackedRel) { + // `hash-object --stdin-paths` accepts newline-delimited input, so it cannot represent a + // pathname containing a newline. Passing an absolute pathname as an argv value keeps every + // legal Git pathname atomic and also works when projectRoot is below the repository root. + const absolutePath = path.resolve(projectRoot, rel); + const pathBytes = Buffer.byteLength(absolutePath, "utf8") + 1; + const wouldExceedBatchLimit = + currentBatch.length && currentBatchBytes + pathBytes > MAX_GIT_HASH_OBJECT_ARGUMENT_BYTES; + if (wouldExceedBatchLimit) { + batches.push(currentBatch); + currentBatch = []; + currentBatchBytes = 0; + } + currentBatch.push(absolutePath); + currentBatchBytes += pathBytes; + } + if (currentBatch.length) batches.push(currentBatch); + + const hashes: string[] = []; + for (const batch of batches) { + const { stdout } = await runGit(projectRoot, ["hash-object", "--", ...batch]); + hashes.push( + ...stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean), + ); + } + return hashes; +} + /** * List files changed in Git. * - base/head: compares commits in the explicit range `${base}..${head ?? "HEAD"}`. diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index 22b66720..fc655ec2 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -891,6 +891,27 @@ describe("Cache invalidation and strict hashing", () => { } }); + it("returns git signatures for tracked paths containing leading whitespace", async () => { + const root = await mkTmpDir("dg-git-sig-special-paths-"); + runGit(root, ["init"]); + runGit(root, ["config", "user.email", "cache@test.local"]); + runGit(root, ["config", "user.name", "Cache Test"]); + + const filePaths = [path.join(root, " leading-and-internal whitespace.ts"), path.join(root, "ordinary.ts")]; + await Promise.all( + filePaths.map((file, index) => fsp.writeFile(file, `export const value${index} = ${index};\n`, "utf8")), + ); + runGit(root, ["add", "-A"]); + runGit(root, ["commit", "-m", "special paths"]); + + const hashes = await gitModule.getGitBlobHashes(root, filePaths); + + expect(hashes.size).toBe(filePaths.length); + for (const filePath of filePaths) { + expect(hashes.get(normalize(filePath))).toMatch(/^[0-9a-f]{40}$/); + } + }); + it("surfaces a genuine git invocation failure instead of silently discarding signatures", async () => { const root = await mkTmpDir("dg-git-sig-invocation-failure-"); // No `git init`: the directory is not a repository, so `git ls-files` genuinely fails diff --git a/tests/native-query-results.test.ts b/tests/native-query-results.test.ts new file mode 100644 index 00000000..9bfa02ac --- /dev/null +++ b/tests/native-query-results.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { buildByteToStringIndexMap } from "../src/native/byteIndex.js"; +import { rangeFromNativeCapture } from "../src/native/queryResults.js"; + +describe("rangeFromNativeCapture", () => { + it("converts UTF-8 byte indexes and point columns to UTF-16 range boundaries", () => { + const source = 'const emoji = "😀";\nconst café = 1;\n'; + const text = "café"; + const startIndex = source.indexOf(text); + const endIndex = startIndex + text.length; + const startByteIndex = Buffer.byteLength(source.slice(0, startIndex), "utf8"); + const endByteIndex = Buffer.byteLength(source.slice(0, endIndex), "utf8"); + const lineStartIndex = source.lastIndexOf("\n", startIndex - 1) + 1; + const startColumn = Buffer.byteLength(source.slice(lineStartIndex, startIndex), "utf8"); + const endColumn = Buffer.byteLength(source.slice(lineStartIndex, endIndex), "utf8"); + + const range = rangeFromNativeCapture( + { + name: "name", + text, + nodeType: "identifier", + start: { row: 1, column: startColumn, index: startByteIndex }, + end: { row: 1, column: endColumn, index: endByteIndex }, + }, + buildByteToStringIndexMap(source), + ); + + expect(range).toEqual({ + start: { line: 2, column: startIndex - lineStartIndex + 1, index: startIndex }, + end: { line: 2, column: endIndex - lineStartIndex + 1, index: endIndex }, + }); + expect(source.slice(range.start.index, range.end.index)).toBe(text); + }); +}); diff --git a/tests/streaming-parser.test.ts b/tests/streaming-parser.test.ts index d57267e1..001fda78 100644 --- a/tests/streaming-parser.test.ts +++ b/tests/streaming-parser.test.ts @@ -195,4 +195,21 @@ index 0000000..1111111 100644 const parsed = parseUnifiedDiff(diffText); expect(parsed.files).toEqual([expect.objectContaining({ path: 'quote"test.ts', kind: "modified" })]); }); + + it("preserves trailing whitespace in an unquoted destination header path", () => { + const pathWithTrailingSpace = `trailing${" "}`; + const diffText = [ + `diff --git a/${pathWithTrailingSpace} b/${pathWithTrailingSpace}`, + "index 0000000..1111111 100644", + `--- a/${pathWithTrailingSpace}`, + `+++ b/${pathWithTrailingSpace}`, + "@@ -0,0 +1 @@", + "+export const value = 1;", + "", + ].join("\n"); + + const parsed = parseUnifiedDiff(diffText); + + expect(parsed.files).toEqual([expect.objectContaining({ path: pathWithTrailingSpace, kind: "modified" })]); + }); }); From 4148f53cb3760127ac44666f4d7264817971de33 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 10:38:21 -0400 Subject: [PATCH 17/29] test: stabilize fixture and identity regressions --- tests/duplicates.test.ts | 4 +--- tests/index.test.ts | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/duplicates.test.ts b/tests/duplicates.test.ts index a0930d92..a68b729e 100644 --- a/tests/duplicates.test.ts +++ b/tests/duplicates.test.ts @@ -2433,9 +2433,7 @@ export function sharedOversizedClone(rows) { const parsed = await ensureParsedContext(displayFile); index.parsed = new Map([[fileIdentityKey(displayFile), parsed]]); - const queryFile = displayFile.includes("Util.ts") - ? displayFile.replace("Util.ts", "util.ts") - : displayFile.replace("util.ts", "Util.ts"); + const queryFile = displayFile.replace(/Util\.ts$/i, "UTIL.ts"); expect(queryFile).not.toBe(displayFile); expect(fileIdentityKey(queryFile)).toBe(fileIdentityKey(displayFile)); expect(index.parsed.has(queryFile)).toBe(false); diff --git a/tests/index.test.ts b/tests/index.test.ts index 437412ac..3793033a 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -180,16 +180,20 @@ describe("Project Indexing", () => { describe("Python Project", () => { it("should index all Python files", async () => { const index = await createTestIndex("python"); - - expectModuleCount(index, 6); - + expectModuleCount(index, 8); const samplePath = path.resolve(process.cwd(), "tests", "samples", "python"); - expectFileInIndex(index, path.join(samplePath, "main.py").replace(/\\/g, "/")); - expectFileInIndex(index, path.join(samplePath, "utils.py").replace(/\\/g, "/")); - expectFileInIndex(index, path.join(samplePath, "helpers.py").replace(/\\/g, "/")); - expectFileInIndex(index, path.join(samplePath, "relative-imports.py").replace(/\\/g, "/")); - expectFileInIndex(index, path.join(samplePath, "__init__.py").replace(/\\/g, "/")); - expectFileInIndex(index, path.join(samplePath, "match_patterns.py").replace(/\\/g, "/")); + for (const file of [ + "main.py", + "utils.py", + "helpers.py", + "relative-imports.py", + "__init__.py", + "match_patterns.py", + ".regressions/unicode_consumer.py", + ".regressions/unicode_def.py", + ]) { + expectFileInIndex(index, path.join(samplePath, file).replace(/\\/g, "/")); + } }); it("should detect Python imports and exports", async () => { From 687deaff1694112e4831e35909ee50fc15383c25 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 11:58:57 -0400 Subject: [PATCH 18/29] fix: parse Unicode identifier continuations --- src/indexer/imports/python.ts | 7 +++++-- src/util/specifiers.ts | 6 +++--- ...mport-extraction-unicode-identifiers.test.ts | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/indexer/imports/python.ts b/src/indexer/imports/python.ts index 4253455d..7ee2b036 100644 --- a/src/indexer/imports/python.ts +++ b/src/indexer/imports/python.ts @@ -118,7 +118,9 @@ export async function collectPythonImportsFromSource(context: PythonImportExtrac } // PEP 3131 permits Unicode identifiers (XID_Start/XID_Continue); an ASCII-only // character class here silently drops every non-ASCII imported name's binding. - const aliasMatch = item.match(/^([\p{L}_][\p{L}\p{N}_]*)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?$/u); + const aliasMatch = item.match( + /^([_\p{XID_Start}][_\p{XID_Continue}]*)(?:\s+as\s+([_\p{XID_Start}][_\p{XID_Continue}]*))?$/u, + ); if (!aliasMatch) continue; const imported = aliasMatch[1]!; const local = aliasMatch[2] ?? imported; @@ -126,7 +128,8 @@ export async function collectPythonImportsFromSource(context: PythonImportExtrac } } - const importPattern = /^(?:\s*)import\s+([\p{L}_][\p{L}\p{N}_.]*)\s*(?:as\s+([\p{L}_][\p{L}\p{N}_]*))?/gmu; + const importPattern = + /^(?:\s*)import\s+([_\p{XID_Start}][.\p{XID_Continue}_]*)\s*(?:as\s+([_\p{XID_Start}][_\p{XID_Continue}]*))?/gmu; for (const match of pySrc.matchAll(importPattern)) { const dotted = match[1]!; const local = match[2] ?? dotted.split(".")[0]!; diff --git a/src/util/specifiers.ts b/src/util/specifiers.ts index c23cf568..ed272c47 100644 --- a/src/util/specifiers.ts +++ b/src/util/specifiers.ts @@ -45,10 +45,10 @@ export function extractJsTsSpecifiers(source: string): ModuleSpecifier[] { const literalMask = buildJsLikeLiteralMask(src); // Capture groups: 1 import-from, 2 side-effect import, 3 export-from, // 4 destructured require, 5 require(), 6 import(), 7 import = require, 8 declare module. - // JS/TS identifiers permit Unicode ID_Start/ID_Continue plus $/_, not just ASCII, so the - // import-equals alias uses \p{L}\p{N} rather than \w (which is ASCII-only). + // JS/TS identifiers permit Unicode ID_Start/ID_Continue plus $/_, with ZWNJ and ZWJ as + // continuation characters, so import-equals aliases must not use ASCII-only \w. const combined = - /^\s*import\s+[^\n;]*?\s+from\s+["']([^"']+)["']|^\s*import\s+["']([^"']+)["']|\bexport\s+[^\n;]*?\s+from\s+["']([^"']+)["']|\b(?:const|let|var)\s*\{[^}]*\}\s*=\s*require\s*\(\s*["']([^"']+)["']\s*\)|(? { expect(extractPythonSpecifiers("from créer import x\n")).toContain("créer"); }); + it("Python import bindings accept combining-mark continuations", async () => { + const bindings: ImportBinding[] = []; + await collectPythonImportsFromSource({ + projectRoot: process.cwd(), + file: path.join(process.cwd(), "consumer.py"), + source: "from package import café as alias\nimport package.café as moduleAlias\n", + pushBinding: (binding) => bindings.push(binding), + }); + + expect(bindings).toEqual([ + expect.objectContaining({ kind: "named", from: "package", imported: "café", local: "alias" }), + expect.objectContaining({ kind: "namespace", from: "package.café", localNS: "moduleAlias" }), + ]); + }); + it("JS CommonJS destructuring require(): Unicode property name binding", async () => { const root = await fsp.mkdtemp(path.join(os.tmpdir(), "cg-cjs-unicode-destructure-")); try { From 7dc31cbb49d6544ab6f55a8fe468b9c5d0935c5e Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 16:27:53 -0400 Subject: [PATCH 19/29] fix: support Unicode import bindings --- src/indexer/build-cache/reports.ts | 4 +- src/indexer/imports/jsTextImports.ts | 37 +++- src/indexer/imports/nativeCaptures.ts | 8 +- src/indexer/imports/python.ts | 17 +- src/languages/importStatementParsers.ts | 16 +- src/native/queryResults.ts | 4 +- src/util/identifiers.ts | 8 + ...allback-import-extraction-messages.test.ts | 34 ++- ...ort-extraction-unicode-identifiers.test.ts | 195 +++++++++++++++--- 9 files changed, 254 insertions(+), 69 deletions(-) create mode 100644 src/util/identifiers.ts diff --git a/src/indexer/build-cache/reports.ts b/src/indexer/build-cache/reports.ts index 0385cc2a..e415353c 100644 --- a/src/indexer/build-cache/reports.ts +++ b/src/indexer/build-cache/reports.ts @@ -117,10 +117,10 @@ export function createFallbackImportExtractionHandler( let message: string; if (event.reason === "reduced-mode") { message = `Native parser unavailable for ${event.language}; using reduced import extraction.`; - } else if (supportsReducedModeRegexRecovery(event.language)) { - message = `Native import recovery degraded for ${event.language}; using native-owned fallback extraction.`; } else if (event.reason === "fast") { message = `Fast mode active for ${event.language}; using regex-based import extraction instead of the native parser.`; + } else if (supportsReducedModeRegexRecovery(event.language)) { + message = `Native import recovery degraded for ${event.language}; using native-owned fallback extraction.`; } else if (event.reason === "query-error") { message = `Native import query failed for ${event.language}; using regex-based fallback extraction.`; } else { diff --git a/src/indexer/imports/jsTextImports.ts b/src/indexer/imports/jsTextImports.ts index 8e75a3ba..b3ca2627 100644 --- a/src/indexer/imports/jsTextImports.ts +++ b/src/indexer/imports/jsTextImports.ts @@ -1,4 +1,5 @@ import { maskJsLikeCommentsStringsAndRegex, stripJsLikeComments } from "../../util/comments.js"; +import { ECMASCRIPT_IDENTIFIER_SOURCE } from "../../util/identifiers.js"; import type { ImportBindingSink, ImportResolver } from "./context.js"; export type JsTextImportExtractionContext = ImportBindingSink & { @@ -7,6 +8,28 @@ export type JsTextImportExtractionContext = ImportBindingSink & { resolveFrom: ImportResolver; }; +const TYPE_NAMED_IMPORT_SPECIFIER_PATTERN = new RegExp( + String.raw`^type\s+(${ECMASCRIPT_IDENTIFIER_SOURCE})(?:\s+as\s+(${ECMASCRIPT_IDENTIFIER_SOURCE}))?$`, + "u", +); +const NAMED_IMPORT_SPECIFIER_PATTERN = new RegExp( + String.raw`^(${ECMASCRIPT_IDENTIFIER_SOURCE})(?:\s+as\s+(${ECMASCRIPT_IDENTIFIER_SOURCE}))?$`, + "u", +); +const NAMESPACE_IMPORT_PATTERN = new RegExp(String.raw`^\*\s+as\s+(${ECMASCRIPT_IDENTIFIER_SOURCE})$`, "u"); +const DEFAULT_REQUIRE_PATTERN = new RegExp( + String.raw`(?:^|[;{}])\s*(?:export\s+)?(?:const|let|var)\s+(${ECMASCRIPT_IDENTIFIER_SOURCE})\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)`, + "gmu", +); +const NAMED_REQUIRE_SPECIFIER_PATTERN = new RegExp( + String.raw`^(${ECMASCRIPT_IDENTIFIER_SOURCE})(?::\s*(${ECMASCRIPT_IDENTIFIER_SOURCE}))?$`, + "u", +); +const IMPORT_EQUALS_REQUIRE_PATTERN = new RegExp( + String.raw`(?:^|[;{}])\s*import\s+(${ECMASCRIPT_IDENTIFIER_SOURCE})\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)`, + "gmu", +); + function sourceForTextImportExtraction(context: JsTextImportExtractionContext): string { if (context.languageId === "ts" || context.languageId === "tsx" || context.languageId === "js") { return stripJsLikeComments(context.source); @@ -24,13 +47,13 @@ function splitNamedImports(namedBlock: string): string[] { function parseNamedImportSpecifier(spec: string): { imported: string; local: string; typeOnly: boolean } | null { // JS/TS identifiers permit Unicode ID_Start/ID_Continue plus $/_, not just ASCII. - const typeOnlyMatch = spec.match(/^type\s+([\p{L}_$][\p{L}\p{N}_$]*)(?:\s+as\s+([\p{L}_$][\p{L}\p{N}_$]*))?$/u); + const typeOnlyMatch = spec.match(TYPE_NAMED_IMPORT_SPECIFIER_PATTERN); if (typeOnlyMatch) { const imported = typeOnlyMatch[1]!; return { imported, local: typeOnlyMatch[2] ?? imported, typeOnly: true }; } - const namedMatch = spec.match(/^([\p{L}_$][\p{L}\p{N}_$]*)(?:\s+as\s+([\p{L}_$][\p{L}\p{N}_$]*))?$/u); + const namedMatch = spec.match(NAMED_IMPORT_SPECIFIER_PATTERN); if (!namedMatch) return null; const imported = namedMatch[1]!; return { imported, local: namedMatch[2] ?? imported, typeOnly: false }; @@ -62,7 +85,7 @@ async function collectEsImports( if (!moduleSpecifier) continue; const typeOnly = typeOnlyImport.test(match[0]); const resolved = await context.resolveFrom(moduleSpecifier); - const namespaceMatch = clause.match(/^\*\s+as\s+([\p{L}_$][\p{L}\p{N}_$]*)$/u); + const namespaceMatch = clause.match(NAMESPACE_IMPORT_PATTERN); if (namespaceMatch) { context.pushBinding({ kind: "namespace", @@ -110,8 +133,7 @@ async function collectCommonJsRequireDeclarations( source: string, maskedSource: string, ): Promise { - const defaultRequirePattern = - /(?:^|[;{}])\s*(?:export\s+)?(?:const|let|var)\s+([\p{L}_$][\p{L}\p{N}_$]*)\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)/gmu; + const defaultRequirePattern = DEFAULT_REQUIRE_PATTERN; for (const match of source.matchAll(defaultRequirePattern)) { if (!matchStartsInCode(maskedSource, match)) continue; const local = match[1]!; @@ -139,7 +161,7 @@ async function collectCommonJsRequireDeclarations( if (!moduleSpecifier) continue; const resolved = await context.resolveFrom(moduleSpecifier); for (const spec of specs) { - const namedMatch = spec.match(/^([\p{L}_$][\p{L}\p{N}_$]*)(?::\s*([\p{L}_$][\p{L}\p{N}_$]*))?$/u); + const namedMatch = spec.match(NAMED_REQUIRE_SPECIFIER_PATTERN); if (!namedMatch) continue; const imported = namedMatch[1]!; const local = namedMatch[2] ?? imported; @@ -160,8 +182,7 @@ async function collectCommonJsImportEquals( source: string, maskedSource: string, ): Promise { - const importEqualsPattern = - /(?:^|[;{}])\s*import\s+([\p{L}_$][\p{L}\p{N}_$]*)\s*=\s*require\s*\(\s*(["'])(?[^"']+)\2\s*\)/gmu; + const importEqualsPattern = IMPORT_EQUALS_REQUIRE_PATTERN; for (const match of source.matchAll(importEqualsPattern)) { if (!matchStartsInCode(maskedSource, match)) continue; const local = match[1]!; diff --git a/src/indexer/imports/nativeCaptures.ts b/src/indexer/imports/nativeCaptures.ts index f6ebcf04..242906fb 100644 --- a/src/indexer/imports/nativeCaptures.ts +++ b/src/indexer/imports/nativeCaptures.ts @@ -1,6 +1,7 @@ import { capturesByName, capturesNamed } from "../../native/queryResults.js"; import type { NativeCapture, NativeMatch } from "../../native/treeSitterNative.js"; import { unquote } from "../../util/ast.js"; +import { ECMASCRIPT_IDENTIFIER_SOURCE } from "../../util/identifiers.js"; import { utf8ByteOffsetToStringIndex } from "../../util/rustTestModules.js"; import { parseGoImportAlias } from "../shared.js"; import type { ImportBinding } from "../types.js"; @@ -17,6 +18,11 @@ type ImportCaptureExtractionContext = { applyStatementOverride: (stmtText: string, typeOnly: boolean, statementStartIndex?: number) => Promise; }; +const OBJECT_PATTERN_BINDING_PATTERN = new RegExp( + String.raw`^(${ECMASCRIPT_IDENTIFIER_SOURCE})(?::\s*(${ECMASCRIPT_IDENTIFIER_SOURCE}))?$`, + "u", +); + function parseObjectPatternBindings(patternText: string): Array<{ imported: string; local: string }> { const trimmed = patternText.trim(); if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return []; @@ -30,7 +36,7 @@ function parseObjectPatternBindings(patternText: string): Array<{ imported: stri for (const part of parts) { const withoutDefault = part.replace(/\s*=\s*.+$/, "").trim(); // JS/TS identifiers permit Unicode ID_Start/ID_Continue plus $/_, not just ASCII. - const match = withoutDefault.match(/^([\p{L}_$][\p{L}\p{N}_$]*)(?::\s*([\p{L}_$][\p{L}\p{N}_$]*))?$/u); + const match = withoutDefault.match(OBJECT_PATTERN_BINDING_PATTERN); if (!match) continue; const imported = match[1]!; const local = match[2] ?? imported; diff --git a/src/indexer/imports/python.ts b/src/indexer/imports/python.ts index 7ee2b036..474bfd0a 100644 --- a/src/indexer/imports/python.ts +++ b/src/indexer/imports/python.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { resolvePythonModule } from "../../util/resolution.js"; import { stripPythonCommentsAndStrings } from "../../util/comments.js"; +import { PYTHON_IDENTIFIER_SOURCE } from "../../util/identifiers.js"; import type { ImportBindingSink, ResolvedImportTarget } from "./context.js"; export type PythonImportExtractionContext = ImportBindingSink & { @@ -105,6 +106,15 @@ async function pushDefaultImport(context: PythonImportExtractionContext, dotted: }); } +const PYTHON_NAMED_IMPORT_PATTERN = new RegExp( + String.raw`^(${PYTHON_IDENTIFIER_SOURCE})(?:\s+as\s+(${PYTHON_IDENTIFIER_SOURCE}))?$`, + "u", +); +const PYTHON_MODULE_IMPORT_PATTERN = new RegExp( + String.raw`^(?:\s*)import\s+(${PYTHON_IDENTIFIER_SOURCE}(?:\.${PYTHON_IDENTIFIER_SOURCE})*)\s*(?:as\s+(${PYTHON_IDENTIFIER_SOURCE}))?`, + "gmu", +); + export async function collectPythonImportsFromSource(context: PythonImportExtractionContext): Promise { const pySrc = stripPythonCommentsAndStrings(context.source); const fromLinePattern = /^\s*from\s+([^\s]+)\s+import\s+([^\n#]+)/gm; @@ -118,9 +128,7 @@ export async function collectPythonImportsFromSource(context: PythonImportExtrac } // PEP 3131 permits Unicode identifiers (XID_Start/XID_Continue); an ASCII-only // character class here silently drops every non-ASCII imported name's binding. - const aliasMatch = item.match( - /^([_\p{XID_Start}][_\p{XID_Continue}]*)(?:\s+as\s+([_\p{XID_Start}][_\p{XID_Continue}]*))?$/u, - ); + const aliasMatch = item.match(PYTHON_NAMED_IMPORT_PATTERN); if (!aliasMatch) continue; const imported = aliasMatch[1]!; const local = aliasMatch[2] ?? imported; @@ -128,8 +136,7 @@ export async function collectPythonImportsFromSource(context: PythonImportExtrac } } - const importPattern = - /^(?:\s*)import\s+([_\p{XID_Start}][.\p{XID_Continue}_]*)\s*(?:as\s+([_\p{XID_Start}][_\p{XID_Continue}]*))?/gmu; + const importPattern = PYTHON_MODULE_IMPORT_PATTERN; for (const match of pySrc.matchAll(importPattern)) { const dotted = match[1]!; const local = match[2] ?? dotted.split(".")[0]!; diff --git a/src/languages/importStatementParsers.ts b/src/languages/importStatementParsers.ts index 43e84d80..9183a32b 100644 --- a/src/languages/importStatementParsers.ts +++ b/src/languages/importStatementParsers.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { XID_IDENTIFIER_SOURCE } from "../util/identifiers.js"; import { isAbsoluteFilePath, normalizePath } from "../util/paths.js"; export type ParsedRustImportStatement = @@ -19,11 +20,18 @@ export type ParsedRustImportStatement = from: string; }; +const RUST_MODULE_PATTERN = new RegExp(String.raw`^mod\s+(${XID_IDENTIFIER_SOURCE})\s*;?$`, "u"); +const RUST_EXTERN_CRATE_PATTERN = new RegExp( + String.raw`^extern\s+crate\s+(${XID_IDENTIFIER_SOURCE})(?:\s+as\s+(${XID_IDENTIFIER_SOURCE}))?\s*;?$`, + "u", +); +const RUST_USE_ALIAS_PATTERN = new RegExp(String.raw`^(.*?)\s+as\s+(${XID_IDENTIFIER_SOURCE})$`, "u"); + export function parseRustImportStatement(stmtText: string): ParsedRustImportStatement | null { const trimmed = stmtText.trim(); // Rust identifiers permit Unicode XID_Start/XID_Continue, not just ASCII. - const modMatch = trimmed.match(/^mod\s+([\p{L}_][\p{L}\p{N}_]*)\s*;?$/u); + const modMatch = trimmed.match(RUST_MODULE_PATTERN); if (modMatch?.[1]) { return { kind: "module", @@ -33,9 +41,7 @@ export function parseRustImportStatement(stmtText: string): ParsedRustImportStat }; } - const externMatch = trimmed.match( - /^extern\s+crate\s+([\p{L}_][\p{L}\p{N}_]*)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?\s*;?$/u, - ); + const externMatch = trimmed.match(RUST_EXTERN_CRATE_PATTERN); if (externMatch?.[1]) { return { kind: "module", @@ -50,7 +56,7 @@ export function parseRustImportStatement(stmtText: string): ParsedRustImportStat if (!useBody) return null; if (useBody.includes("{") || useBody.includes(",")) return null; - const aliasMatch = useBody.match(/^(.*?)\s+as\s+([\p{L}_][\p{L}\p{N}_]*)$/u); + const aliasMatch = useBody.match(RUST_USE_ALIAS_PATTERN); const rawPath = aliasMatch?.[1]?.trim() ?? useBody; const alias = aliasMatch?.[2]; diff --git a/src/native/queryResults.ts b/src/native/queryResults.ts index 2e64ccf7..3c361e3b 100644 --- a/src/native/queryResults.ts +++ b/src/native/queryResults.ts @@ -15,9 +15,9 @@ export function capturesNamed(match: NativeMatch, name: string): NativeCapture[] } /** - * Rust's Tree-sitter captures expose UTF-8 byte offsets. `Range` and every downstream + * Native Tree-sitter captures use UTF-8 byte offsets. `Range` and every downstream * consumer (source slicing, portable handles, rename edits) expect UTF-16 string indexes, - * so every capture must convert through the caller's per-file `byteIndexMap` here. + * so every native capture converts through the caller's per-file `byteIndexMap` here. */ export function rangeFromNativeCapture(capture: NativeCapture, byteIndexMap: ByteToStringIndexMap): Range { const startPosition = stringPositionForBytePoint(byteIndexMap, capture.start); diff --git a/src/util/identifiers.ts b/src/util/identifiers.ts new file mode 100644 index 00000000..65904c64 --- /dev/null +++ b/src/util/identifiers.ts @@ -0,0 +1,8 @@ +/** ECMAScript identifier syntax, including ZWNJ and ZWJ continuation characters. */ +export const ECMASCRIPT_IDENTIFIER_SOURCE = String.raw`[$_\p{ID_Start}](?:[$_\p{ID_Continue}]|\u200c|\u200d)*`; + +/** Unicode XID identifiers, with underscores permitted at every position. */ +export const XID_IDENTIFIER_SOURCE = String.raw`[_\p{XID_Start}][_\p{XID_Continue}]*`; + +/** Python identifiers use normalized Unicode XID properties (PEP 3131). */ +export const PYTHON_IDENTIFIER_SOURCE = XID_IDENTIFIER_SOURCE; diff --git a/tests/fallback-import-extraction-messages.test.ts b/tests/fallback-import-extraction-messages.test.ts index 6974501a..75ae87b3 100644 --- a/tests/fallback-import-extraction-messages.test.ts +++ b/tests/fallback-import-extraction-messages.test.ts @@ -2,6 +2,10 @@ import { describe, it, expect, vi } from "vitest"; import { createFallbackImportExtractionHandler } from "../src/indexer/build-cache/reports.js"; import type { FallbackImportExtractionReason } from "../src/graphs/specifiers.js"; +const logMocks = vi.hoisted(() => ({ logWithLevel: vi.fn() })); + +vi.mock("../src/logging.js", () => ({ logWithLevel: logMocks.logWithLevel })); + describe("Fallback import extraction human messages (D11)", () => { const cases: Array<{ reason: FallbackImportExtractionReason; language: string; expectSubstring: string }> = [ // CSS has no regex-recovery support baked into the native layer, so these reasons @@ -9,30 +13,24 @@ describe("Fallback import extraction human messages (D11)", () => { { reason: "query-empty", language: "css", expectSubstring: "returned no results" }, { reason: "query-error", language: "css", expectSubstring: "query failed" }, { reason: "fast", language: "css", expectSubstring: "Fast mode active" }, + { reason: "fast", language: "ts", expectSubstring: "Fast mode active" }, ]; it.each(cases)( "gives a human sentence for reason=$reason, language=$language", ({ reason, language, expectSubstring }) => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - const debugSpy = vi.spyOn(console, "debug").mockImplementation(() => {}); - try { - const handler = createFallbackImportExtractionHandler(undefined, { logLevel: "debug" }); - handler?.({ language, reason, file: "styles.css" }); - - const allCalls = [...warnSpy.mock.calls, ...debugSpy.mock.calls]; - expect(allCalls).toHaveLength(1); - const message = String(allCalls[0]?.[0] ?? ""); + logMocks.logWithLevel.mockClear(); + const handler = createFallbackImportExtractionHandler(undefined, { logLevel: "debug" }); + handler?.({ language, reason, file: "styles.css" }); - // Regression guard for the exact bare label observed on stderr (V7): - // "Regex fallback import extraction { language: 'css', reason: 'query-empty' }" - expect(message).not.toBe("Regex fallback import extraction"); - expect(message).toContain(language); - expect(message).toContain(expectSubstring); - } finally { - warnSpy.mockRestore(); - debugSpy.mockRestore(); - } + expect(logMocks.logWithLevel).toHaveBeenCalledTimes(1); + const [logLevel, severity, message] = logMocks.logWithLevel.mock.calls[0] ?? []; + expect(logLevel).toBe("debug"); + if (reason === "fast") expect(severity).toBe("debug"); + expect(message).toBeTypeOf("string"); + expect(message).not.toBe("Regex fallback import extraction"); + expect(message).toContain(language); + expect(message).toContain(expectSubstring); }, ); }); diff --git a/tests/import-extraction-unicode-identifiers.test.ts b/tests/import-extraction-unicode-identifiers.test.ts index dde74ca8..4eada37b 100644 --- a/tests/import-extraction-unicode-identifiers.test.ts +++ b/tests/import-extraction-unicode-identifiers.test.ts @@ -9,10 +9,14 @@ import { parsePhpImportStatement, parseRustImportStatement, } from "../src/languages/importStatementParsers.js"; -import { extractPythonSpecifiers } from "../src/util.js"; +import { extractJsTsSpecifiers, extractPythonSpecifiers } from "../src/util.js"; import { buildProjectIndex } from "../src/index.js"; +import { collectJsTextImports } from "../src/indexer/imports/jsTextImports.js"; +import { collectNativeCaptureImportBindings } from "../src/indexer/imports/nativeCaptures.js"; +import { finalizeLanguageSpecificImports } from "../src/indexer/imports/languageSpecific.js"; import { collectPythonImportsFromSource } from "../src/indexer/imports/python.js"; import type { ImportBinding } from "../src/indexer/types.js"; +import type { NativeMatch } from "../src/native/treeSitterNative.js"; // C11-adjacent finding: several import/alias extractors used an ASCII-only [A-Za-z_][\w]* // character class, which silently drops the binding (or the whole statement) for any @@ -21,23 +25,23 @@ import type { ImportBinding } from "../src/indexer/types.js"; // letters, Go's Unicode "letter" production, JS/TS ID_Start/ID_Continue, PEP 3131 Python). describe("Import/alias extraction accepts Unicode identifiers", () => { it("Rust: extern crate alias, use alias, and module name", () => { - expect(parseRustImportStatement("mod créer;")).toEqual({ + expect(parseRustImportStatement("mod \u2118\u0301;")).toEqual({ kind: "module", - from: "créer", - local: "créer", + from: "\u2118\u0301", + local: "\u2118\u0301", isExternCrate: false, }); - expect(parseRustImportStatement("extern crate créer as créé;")).toEqual({ + expect(parseRustImportStatement("extern crate \u2118 as alias\u0301;")).toEqual({ kind: "module", - from: "créer", - local: "créé", + from: "\u2118", + local: "alias\u0301", isExternCrate: true, }); - expect(parseRustImportStatement("use std::foo as créer;")).toEqual({ + expect(parseRustImportStatement("use std::foo as alias\u0301;")).toEqual({ kind: "member", from: "std", imported: "foo", - local: "créer", + local: "alias\u0301", }); }); @@ -119,24 +123,159 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { await fsp.rm(root, { recursive: true, force: true }); } }); + + it("JS text fallback preserves every Unicode identifier import form", async () => { + const bindings: ImportBinding[] = []; + await collectJsTextImports({ + source: [ + 'import { \u2118 as namedAlias\u200c, type typeName\u200d as typeAlias } from \"es\";', + 'import * as namespaceAlias\u200d from \"namespace\";', + 'const defaultAlias\u200c = require(\"default\");', + 'const { \u2118: objectAlias\u200d, propertyName\u200c } = require(\"properties\");', + 'import equalsAlias\u200d = require(\"equals\");', + ].join("\n"), + languageId: "ts", + resolveFrom: async (from) => ({ external: from }), + pushBinding: (binding) => bindings.push(binding), + }); + + expect(bindings).toEqual([ + { + kind: "named", + local: "namedAlias\u200c", + imported: "\u2118", + from: "es", + resolved: { external: "es" }, + typeOnly: false, + }, + { + kind: "named", + local: "typeAlias", + imported: "typeName\u200d", + from: "es", + resolved: { external: "es" }, + typeOnly: true, + }, + { + kind: "namespace", + localNS: "namespaceAlias\u200d", + from: "namespace", + resolved: { external: "namespace" }, + typeOnly: false, + }, + { + kind: "default", + local: "defaultAlias\u200c", + from: "default", + resolved: { external: "default" }, + mechanism: "cjs", + }, + { + kind: "named", + local: "objectAlias\u200d", + imported: "\u2118", + from: "properties", + resolved: { external: "properties" }, + mechanism: "cjs", + }, + { + kind: "named", + local: "propertyName\u200c", + imported: "propertyName\u200c", + from: "properties", + resolved: { external: "properties" }, + mechanism: "cjs", + }, + { + kind: "default", + local: "equalsAlias\u200d", + from: "equals", + resolved: { external: "equals" }, + mechanism: "cjs", + }, + ]); + }); }); -// Two additional sibling fixes were made for the same defect class but could not be proven -// to change observable behavior against this codebase's actual extraction pipeline, so they -// are documented here rather than asserted as regression tests: -// -// - src/util/specifiers.ts's combined JS/TS regex (import X = require(...) alternative): the -// module specifier is still recovered via the separate bare `require(...)` alternative in -// the same combined pattern even when the "import X =" alias fails to match, and -// extractJsTsSpecifiers's return type never exposes the alias/local name at all. The fix -// (Unicode-aware alias class) is still correct and removes a latent dependency on that -// alternative-pattern fallback, but there is no independently observable before/after -// difference through this function's public contract. -// - src/indexer/imports/languageSpecific.ts's normalizeGoImports alias regex: Go's native -// query captures already expose the import alias identifier as raw capture text (unaffected -// by any JS regex), so a named Unicode alias is already correct via the primary native path -// before this fix. The only alias values normalizeGoImports's regex result changes behavior -// for are the single-character "." (dot-import) and "_" (blank-import) sentinels, both of -// which were already covered by the old ASCII character class. The fix is still correct -// (defense in depth against a change to the native query), but not independently provable -// as a behavior change in this codebase today. +describe("Unicode import parser seams", () => { + it("parses native object-pattern captures with ECMAScript-only identifier characters", async () => { + const bindings: ImportBinding[] = []; + const source = "const { \u2118: localAlias\u200d } = require('properties');"; + const point = { row: 0, column: 0, index: 0 }; + const match: NativeMatch = { + patternIndex: 0, + captures: [ + { name: "from", text: "'properties'", nodeType: "string", start: point, end: point }, + { name: "pattern", text: "{ \u2118: localAlias\u200d }", nodeType: "object_pattern", start: point, end: point }, + ], + }; + const resolveFrom = async (from: string) => ({ external: from }); + const pushBinding = (binding: ImportBinding) => bindings.push(binding); + const getBindings = () => bindings; + const replaceBindings = (next: ImportBinding[]) => bindings.splice(0, bindings.length, ...next); + + await collectNativeCaptureImportBindings( + { + source, + languageId: "ts", + isTypeOnly: () => false, + resolveFrom, + pushBinding, + languageContext: { + file: "consumer.ts", + projectRoot: process.cwd(), + source, + languageId: "ts", + resolveFrom, + pushBinding, + getBindings, + replaceBindings, + }, + applyStatementOverride: async () => false, + }, + [match], + ); + + expect(bindings).toEqual([ + { + kind: "named", + local: "localAlias\u200d", + imported: "\u2118", + from: "properties", + resolved: { external: "properties" }, + typeOnly: false, + }, + ]); + }); + + it("normalizes a Unicode Go import alias from text", async () => { + const bindings: ImportBinding[] = [ + { kind: "namespace", localNS: "fallback", from: "example.test/dep", resolved: { external: "example.test/dep" } }, + ]; + const resolveFrom = async (from: string) => ({ external: from }); + const pushBinding = (binding: ImportBinding) => bindings.push(binding); + const getBindings = () => bindings; + const replaceBindings = (next: ImportBinding[]) => bindings.splice(0, bindings.length, ...next); + + await finalizeLanguageSpecificImports({ + file: "consumer.go", + projectRoot: process.cwd(), + source: 'import \u4e2d "example.test/dep"', + languageId: "go", + resolveFrom, + pushBinding, + getBindings, + replaceBindings, + }); + + expect(bindings).toEqual([ + { kind: "namespace", localNS: "\u4e2d", from: "example.test/dep", resolved: { external: "example.test/dep" } }, + ]); + }); + + it("extracts Unicode import-equals bindings in the specifier fallback", () => { + expect(extractJsTsSpecifiers("import alias\u200d = require('package');\n")).toEqual([ + { spec: "package", exportCondition: "require" }, + ]); + }); +}); From 7d7947b276b8c63afa0c81c440f074b1bffa36a3 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 17:12:24 -0400 Subject: [PATCH 20/29] refactor: extract shared decodeGitPath utility to util/git Move the C-style Git path decoder out of impact/parse.ts into util/git.ts so it can be reused by other Git path handling in this module, and cover it directly plus copy-path header decoding in the unified-diff parser. --- docs/coverage/js.md | 8 ++--- src/impact/parse.ts | 55 +------------------------------- src/util/git.ts | 49 ++++++++++++++++++++++++++++ tests/git-diff-semantics.test.ts | 20 ++++++++++++ tests/streaming-parser.test.ts | 17 ++++++++++ 5 files changed, 91 insertions(+), 58 deletions(-) diff --git a/docs/coverage/js.md b/docs/coverage/js.md index 36ca843f..e8ecd39e 100644 --- a/docs/coverage/js.md +++ b/docs/coverage/js.md @@ -6,9 +6,9 @@ Source: `coverage/js/lcov.info` | Metric | Hit | Found | Coverage | | --------- | ----: | ----: | -------: | -| Lines | 27353 | 30141 | 90.75% | -| Functions | 4559 | 4839 | 94.21% | -| Branches | 20655 | 26070 | 79.23% | +| Lines | 27455 | 30228 | 90.83% | +| Functions | 4563 | 4842 | 94.24% | +| Branches | 20694 | 26098 | 79.29% | ## Least-covered Files @@ -43,6 +43,7 @@ These files have line records but no function or branch records, so they are tra | --------------------------------------- | ------: | --------: | -------: | | `src/cliBootstrap.ts` | 0.00% | n/a | n/a | | `src/languages/definitions/jsFamily.ts` | 100.00% | n/a | n/a | +| `src/util/identifiers.ts` | 100.00% | n/a | n/a | | `src/duplicate-keywords.ts` | 100.00% | n/a | n/a | | `src/impact/types.ts` | 100.00% | n/a | n/a | | `src/languages/definitions/adoc.ts` | 100.00% | n/a | n/a | @@ -50,6 +51,5 @@ These files have line records but no function or branch records, so they are tra | `src/languages/definitions/hbs.ts` | 100.00% | n/a | n/a | | `src/languages/definitions/markdown.ts` | 100.00% | n/a | n/a | | `src/languages/definitions/mdx.ts` | 100.00% | n/a | n/a | -| `src/languages/definitions/rst.ts` | 100.00% | n/a | n/a | Generated from LCOV by `node ./scripts/coverage-markdown.mjs`. diff --git a/src/impact/parse.ts b/src/impact/parse.ts index 68b5ddee..9565538f 100644 --- a/src/impact/parse.ts +++ b/src/impact/parse.ts @@ -1,5 +1,6 @@ import { Readable } from "node:stream"; import { StringDecoder } from "node:string_decoder"; +import { decodeGitPath } from "../util/git.js"; import type { Diff, FileChange, Hunk } from "./types.js"; type ParsedFileChange = FileChange & { @@ -131,60 +132,6 @@ function decodeStreamChunk(decoder: StringDecoder, chunk: unknown): string { return String(chunk); } -function decodeGitPath(rawPath: string): string { - if (!rawPath.startsWith('"') || !rawPath.endsWith('"')) { - return rawPath; - } - - // Git quotes a path when it contains non-ASCII or special bytes, escaping each raw byte - // independently as `\NNN` (octal). A multi-byte UTF-8 character becomes several consecutive - // `\NNN` escapes that must be recombined as raw bytes and decoded together as UTF-8 - - // decoding each escape as its own code point (the previous approach) mojibakes every - // non-ASCII path (e.g. "café.ts" became "café.ts"). - const inner = rawPath.slice(1, -1); - const bytes: number[] = []; - for (let index = 0; index < inner.length; ) { - const char = inner[index]!; - if (char !== "\\") { - const codePoint = inner.codePointAt(index)!; - bytes.push(...Buffer.from(String.fromCodePoint(codePoint), "utf8")); - index += codePoint > 0xffff ? 2 : 1; - continue; - } - const octal = inner.slice(index + 1, index + 4).match(/^[0-7]{1,3}/); - if (octal) { - bytes.push(parseInt(octal[0], 8) & 0xff); - index += 1 + octal[0].length; - continue; - } - const next = inner[index + 1]; - if (next === "\\" || next === '"') { - bytes.push(next.charCodeAt(0)); - index += 2; - continue; - } - if (next === "n") { - bytes.push(0x0a); - index += 2; - continue; - } - if (next === "r") { - bytes.push(0x0d); - index += 2; - continue; - } - if (next === "t") { - bytes.push(0x09); - index += 2; - continue; - } - // Unrecognized escape: keep the backslash literally. - bytes.push(0x5c); - index += 1; - } - return Buffer.from(bytes).toString("utf8"); -} - function stripDiffGitPrefix(pathValue: string, prefix: "a/" | "b/"): string { return pathValue.startsWith(prefix) ? pathValue.slice(prefix.length) : pathValue; } diff --git a/src/util/git.ts b/src/util/git.ts index 6a4d9562..cae171e9 100644 --- a/src/util/git.ts +++ b/src/util/git.ts @@ -16,6 +16,55 @@ const MAX_GIT_HASH_OBJECT_ARGUMENT_BYTES = 24 * 1024; let gitExecutableForTests: string | null = null; +/** Decodes Git's optional C-style quoted pathname representation without trimming legal path bytes. */ +export function decodeGitPath(rawPath: string): string { + if (!rawPath.startsWith('"') || !rawPath.endsWith('"')) { + return rawPath; + } + + const inner = rawPath.slice(1, -1); + const bytes: number[] = []; + for (let index = 0; index < inner.length; ) { + const char = inner[index]!; + if (char !== "\\") { + const codePoint = inner.codePointAt(index)!; + bytes.push(...Buffer.from(String.fromCodePoint(codePoint), "utf8")); + index += codePoint > 0xffff ? 2 : 1; + continue; + } + const octal = inner.slice(index + 1, index + 4).match(/^[0-7]{1,3}/); + if (octal) { + bytes.push(parseInt(octal[0], 8) & 0xff); + index += 1 + octal[0].length; + continue; + } + const next = inner[index + 1]; + if (next === "\\" || next === '"') { + bytes.push(next.charCodeAt(0)); + index += 2; + continue; + } + if (next === "n") { + bytes.push(0x0a); + index += 2; + continue; + } + if (next === "r") { + bytes.push(0x0d); + index += 2; + continue; + } + if (next === "t") { + bytes.push(0x09); + index += 2; + continue; + } + bytes.push(0x5c); + index += 1; + } + return Buffer.from(bytes).toString("utf8"); +} + /** Test-only override of the Git executable path. Pass null to restore. */ export function setGitExecutableForTests(executable: string | null): void { gitExecutableForTests = executable; diff --git a/tests/git-diff-semantics.test.ts b/tests/git-diff-semantics.test.ts index c453e039..0ff31e59 100644 --- a/tests/git-diff-semantics.test.ts +++ b/tests/git-diff-semantics.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { listChangedFiles, listUntrackedFiles, getUnifiedDiff } from "../src/util.js"; +import { decodeGitPath } from "../src/util/git.js"; import { parseUnifiedDiff } from "../src/impact/parse.js"; import { runGit as git } from "./helpers/git.js"; @@ -252,6 +253,25 @@ describe("git diff semantics: rename detection is deterministic regardless of us }); }); +describe("Git C-style quoted path decoding", () => { + it("preserves unquoted paths and decodes supported quote escapes", () => { + const cases = [ + ["unquoted path with trailing ", "unquoted path with trailing "], + ['"café.ts"', "café.ts"], + ['"caf\\303\\251.ts"', "café.ts"], + ['"emoji \\360\\237\\230\\200.ts"', "emoji 😀.ts"], + ['"quote\\" and slash\\\\.ts"', 'quote" and slash\\.ts'], + ['"tab\\tline\\ncarriage\\r.ts"', "tab\tline\ncarriage\r.ts"], + ['"\\1\\12\\123"', "\x01\nS"], + ['"unknown\\qtrailing\\"', "unknown\\qtrailing\\"], + ]; + + for (const [rawPath, expected] of cases) { + expect(decodeGitPath(rawPath)).toBe(expected); + } + }); +}); + describe("listUntrackedFiles", () => { it("lists new files Git has not been told to track", async () => { const root = await makeGitTempDir("codegraph-git-untracked-"); diff --git a/tests/streaming-parser.test.ts b/tests/streaming-parser.test.ts index 001fda78..9626a6be 100644 --- a/tests/streaming-parser.test.ts +++ b/tests/streaming-parser.test.ts @@ -212,4 +212,21 @@ index 0000000..1111111 100644 expect(parsed.files).toEqual([expect.objectContaining({ path: pathWithTrailingSpace, kind: "modified" })]); }); + + it("decodes escaped copy paths across diff and file headers", () => { + const diffText = `diff --git "a/source\\t.ts" "b/copied\\t.ts" +similarity index 100% +copy from "source\\t.ts" +copy to "copied\\t.ts" +--- "a/source\\t.ts" ++++ "b/copied\\t.ts" +@@ -1 +1 @@ +-old ++new +`; + + expect(parseUnifiedDiff(diffText).files).toEqual([ + expect.objectContaining({ kind: "added", path: "copied\t.ts", oldPath: "source\t.ts" }), + ]); + }); }); From 3f51c094b826527cd3107a0d384a7534ee4151dd Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 17:41:43 -0400 Subject: [PATCH 21/29] fix: address review feedback on Git path decoding and identifier breadth - decodeGitPath: add missing \a, \b, \v, \f C-style escapes so Git paths using those control-byte escapes decode to the real filename instead of leaving the backslash literal. - runGit: decode stdout/stderr incrementally with StringDecoder instead of chunk.toString() per Buffer, so a multibyte UTF-8 sequence split across two stdout chunks no longer decodes to U+FFFD on each half. - Python native-query and text-fallback module/alias extraction (in graphs/specifiers.ts and util/specifiers.ts) now match PEP 3131 XID_Start/XID_Continue per dotted segment, fixing both a dropped combining-mark continuation and a digit incorrectly allowed right after a '.' separator. - PHP use-clause alias parsing now accepts any byte >= 0x80 at any position (PHP's actual identifier rule) via a new PHP_IDENTIFIER_SOURCE constant, instead of the narrower \p{L}/\p{N} class that dropped non-letter aliases such as emoji. - Document the broadened identifier support in language-parity.md and scenario-catalog.md per repo convention. --- docs/coverage/js.md | 6 +-- docs/language-parity.md | 1 + docs/scenario-catalog.md | 17 ++++--- src/graphs/specifiers.ts | 18 ++++++-- src/languages/importStatementParsers.ts | 9 ++-- src/util/git.ts | 44 ++++++++++++------- src/util/identifiers.ts | 7 +++ src/util/specifiers.ts | 14 ++++-- tests/git-diff-semantics.test.ts | 38 +++++++++++++++- ...ort-extraction-unicode-identifiers.test.ts | 36 +++++++++++++++ 10 files changed, 154 insertions(+), 36 deletions(-) diff --git a/docs/coverage/js.md b/docs/coverage/js.md index e8ecd39e..56955163 100644 --- a/docs/coverage/js.md +++ b/docs/coverage/js.md @@ -6,9 +6,9 @@ Source: `coverage/js/lcov.info` | Metric | Hit | Found | Coverage | | --------- | ----: | ----: | -------: | -| Lines | 27455 | 30228 | 90.83% | +| Lines | 27458 | 30231 | 90.83% | | Functions | 4563 | 4842 | 94.24% | -| Branches | 20694 | 26098 | 79.29% | +| Branches | 20693 | 26098 | 79.29% | ## Least-covered Files @@ -42,8 +42,8 @@ These files have line records but no function or branch records, so they are tra | File | Lines | Functions | Branches | | --------------------------------------- | ------: | --------: | -------: | | `src/cliBootstrap.ts` | 0.00% | n/a | n/a | -| `src/languages/definitions/jsFamily.ts` | 100.00% | n/a | n/a | | `src/util/identifiers.ts` | 100.00% | n/a | n/a | +| `src/languages/definitions/jsFamily.ts` | 100.00% | n/a | n/a | | `src/duplicate-keywords.ts` | 100.00% | n/a | n/a | | `src/impact/types.ts` | 100.00% | n/a | n/a | | `src/languages/definitions/adoc.ts` | 100.00% | n/a | n/a | diff --git a/docs/language-parity.md b/docs/language-parity.md index f1035a8b..74d42302 100644 --- a/docs/language-parity.md +++ b/docs/language-parity.md @@ -92,6 +92,7 @@ Notes: - JavaScript expands `module.exports = { ...source }` for statically resolvable CommonJS imports and local object literals. Dynamic spread sources remain explicit namespace-reexport markers rather than silently disappearing. - Node `package.json#exports` condition matching follows author key order with mutually exclusive `import`/`require` modes threaded from ESM `import`/`import()` versus CommonJS `require()` (and TypeScript `import x = require(...)`). Nested conditions, array fallbacks, and `default` termination match Node for those cases. Custom `--conditions`, `browser`/`types`/`development`/`production`, import attributes, and `#imports` maps are not modeled. - Ruby treats `Constant = Struct.new(...)` as a class-kind symbol and a synthetic detailed class declaration. Runtime-computed class factories remain outside this recognition. +- Import and alias binding extraction accepts each source language's real identifier grammar rather than an ASCII-narrowed approximation: JS/TS/TSX use ID*Start/ID_Continue plus `$`/`*`and ZWNJ/ZWJ continuations, Python uses PEP 3131 XID_Start/XID_Continue, Rust/Go/Java/Kotlin use their Unicode letter/XID identifier rules, and PHP accepts any byte`>= 0x80` at any identifier position. This covers native-query statement parsing, text/regex fallback recovery, and specifier extraction for those languages. ## Project file discovery coverage diff --git a/docs/scenario-catalog.md b/docs/scenario-catalog.md index 5b5f1019..49484416 100644 --- a/docs/scenario-catalog.md +++ b/docs/scenario-catalog.md @@ -113,6 +113,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Embedded struct fields | `tests/samples/go/embedding.go`, `tests/languages/go.test.ts`, `tests/goto.test.ts`, `tests/references.test.ts` | Struct field declarations, direct selector reads, and promoted selector reads resolve to the field and appear in references. | Internal regression fixture | 2026-08-11 | | Type-position locals exclusion | `tests/samples/go/contracts.go`, `tests/languages/go.test.ts` | Exact symbol extraction keeps params, short vars, receivers, fields, methods, and type specs while excluding type-position identifiers such as builtin `int` and generic type parameter `T`. | Internal regression fixture | 2026-08-11 | | Range variables and blank identifiers | `tests/samples/go/range-variables.go`, `tests/languages/go.test.ts`, `tests/goto.test.ts`, `tests/references.test.ts` | `for i, v := range xs` indexes and resolves `i` and `v`; `_` remains non-navigable. | Internal regression fixture | 2026-08-11 | +| Unicode import alias | `tests/import-extraction-unicode-identifiers.test.ts` | A Go import alias accepts any Unicode letter per the Go spec's "letter" production, in both native-query and text-fallback recovery. | Internal regression test | 2026-08-16 | ## HTML @@ -172,6 +173,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Static wildcard imports | `tests/samples/java/StaticWildcardImports.java`, `tests/samples/java/utils/Utils.java` | Dependency graph, go-to-definition, and references resolve Java static wildcard imports back to the declaring utility class. | Internal regression fixture | 2026-03-29 | | Nested classes and interfaces | `tests/samples/java/NestedTypes.java` | Symbol extraction includes nested classes, nested interfaces, and their member methods. | Internal regression fixture | 2026-03-22 | | `record` declarations | `tests/samples/java/RecordTypes.java` | Symbol extraction indexes record declarations as class-kind symbols alongside plain classes, and record `implements` interface conformance participates in type hierarchy the same as an ordinary class. | Internal regression fixture | 2026-08-09 | +| Unicode import identifiers | `tests/import-extraction-unicode-identifiers.test.ts` | `import` and static `import` statements resolve Unicode-named classes and members through the shared Java Unicode-letter identifier rule, in both native-query and text-fallback recovery. | Internal regression test | 2026-08-16 | ## JavaScript @@ -183,6 +185,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Class field navigation | `tests/class-field-locals.test.ts` | Public and `#private` class field declarations are indexed as navigable variable definitions; `goto` resolves constructed-receiver field access (`const w = new Widget(); w.size`) to the field declaration. | Internal regression test | 2026-08-09 | | CommonJS spread exports | `tests/samples/language-regressions/javascript/*.js`, `tests/languages/javascript.test.ts` | `module.exports = { ...base, ...local }` reexports static `require()` and local-object members; a dynamic spread retains an explicit uncertainty marker. | Internal regression fixture | 2026-08-11 | | Conditional package exports order | `tests/package-exports.test.ts`, `tests/node-resolution.test.ts` | `exports` conditions evaluate in author key order (`node` before `import` flips with key order); `require()` consumers resolve the `require` target and ESM `import` consumers resolve the `import` target for dual `.cjs`/`.mjs` packages, including nested conditions and `default` termination. | Internal regression test | 2026-08-11 | +| Unicode import identifiers | `tests/import-extraction-unicode-identifiers.test.ts` | CommonJS destructuring, `import =`/`require()` equals bindings, and text-fallback import/alias extraction accept full ID*Start/ID_Continue identifiers (including `$`/`*` and ZWNJ/ZWJ continuations), not just ASCII/letter-digit subsets. | Internal regression test | 2026-08-16 | ## Kotlin @@ -193,6 +196,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Enums, type aliases, and top-level properties | `tests/samples/kotlin/Models.kt` | Symbol extraction includes enum declarations and enum entries, type aliases, top-level properties, and generic classes. | Internal regression fixture | 2026-03-22 | | Package go-to-definition | `tests/goto.test.ts` | Imported top-level functions and imported classes resolve from `main.kt` into `utils/helperFunction.kt`. | Internal regression test | 2026-03-23 | | Package references | `tests/references.test.ts` | Imported function and class references resolve across `main.kt` and `utils/helperFunction.kt`. | Internal regression test | 2026-03-23 | +| Unicode import alias | `tests/import-extraction-unicode-identifiers.test.ts` | `import ... as alias` accepts a full Unicode identifier alias, not just ASCII letters/digits. | Internal regression test | 2026-08-16 | ## LESS @@ -203,11 +207,12 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. ## Python -| Scenario | Sample | Expected behavior | Source | Date added | -| ------------------------------- | ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | ---------- | -| Relative `from` imports | `tests/samples/python/relative-imports.py` | Dependency graph includes edges to `utils.py` and `helpers.py` for relative `from` imports. | https://github.com/tree-sitter/tree-sitter-python | 2026-01-22 | -| `__all__` export filtering | `tests/languages/python.test.ts` | Export extraction respects `__all__` tuple/list assignments and avoids false positives from nearby strings. | Internal regression test | 2026-03-22 | -| Match bindings and `.pyi` stubs | `tests/samples/language-regressions/python/*`, `tests/languages/python.test.ts` | Tuple and `as` pattern captures are navigable locals with references, and `.pyi` files are discovered and their class/function symbols are indexed. | Internal regression fixture | 2026-08-11 | +| Scenario | Sample | Expected behavior | Source | Date added | +| ------------------------------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------- | ---------- | +| Relative `from` imports | `tests/samples/python/relative-imports.py` | Dependency graph includes edges to `utils.py` and `helpers.py` for relative `from` imports. | https://github.com/tree-sitter/tree-sitter-python | 2026-01-22 | +| `__all__` export filtering | `tests/languages/python.test.ts` | Export extraction respects `__all__` tuple/list assignments and avoids false positives from nearby strings. | Internal regression test | 2026-03-22 | +| Match bindings and `.pyi` stubs | `tests/samples/language-regressions/python/*`, `tests/languages/python.test.ts` | Tuple and `as` pattern captures are navigable locals with references, and `.pyi` files are discovered and their class/function symbols are indexed. | Internal regression fixture | 2026-08-11 | +| Unicode module names | `tests/import-extraction-unicode-identifiers.test.ts` | `import`/`from` module and alias names accept full PEP 3131 XID_Start/XID_Continue identifiers, including combining-mark continuations, in native-query parsing and text-fallback specifier extraction. A dotted segment must itself start with an identifier character, so a digit cannot immediately follow a `.` separator. | Internal regression test | 2026-08-16 | ## PHP @@ -226,6 +231,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Exact declaration symbols | `tests/languages/php.test.ts` | PHP symbol sets include real declaration names, enum cases, constants, and declared properties, while namespace names and ordinary variable uses are excluded. | Internal parity fixture | 2026-08-11 | | Typed, untyped, and static properties | `tests/samples/php/properties.php`, `tests/languages/php.test.ts`, `tests/goto.test.ts`, `tests/references.test.ts` | Property declarations are indexed, and `$this->property` plus static-property uses navigate to their declared property and appear in references. | Internal regression fixture | 2026-08-11 | | Enum interface conformance | `tests/samples/php/EnumImplementation.php`, `tests/languages/php.test.ts` | PHP 8.1 enums using `implements` emit a detailed `implements` edge and appear in implementation lookup for the interface. | Internal regression fixture | 2026-08-11 | +| Unicode `use` alias | `tests/import-extraction-unicode-identifiers.test.ts` | `use ... as alias` accepts any byte `>= 0x80` at any position, matching PHP's real identifier rule rather than a narrower Unicode-letter/digit subset; covers both plain and grouped `use` clauses. | Internal regression test | 2026-08-16 | ## Ruby @@ -245,6 +251,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Reexports and nested modules | `tests/samples/rust/reexports.rs`, `tests/samples/rust/nested.rs`, `tests/samples/rust/nested_service.rs` | Go-to-definition and references cover nested-module type resolution, and native semantic coverage keeps those nested/reexport fixtures stable. | Internal regression fixture | 2026-03-23 | | Traits, impls, enums, and exported types | `tests/samples/rust/models.rs` | Symbol extraction includes trait declarations, enum declarations, enum variants, impl-backed methods, and exported structs from Rust modules. | Internal regression fixture | 2026-03-22 | | `macro_rules!` definitions | `tests/samples/rust/.regressions/macros.rs`, `tests/languages/rust.test.ts` | Macro definitions are chunked and indexed; macro invocations resolve to the definition and count as references. | Internal regression fixture | 2026-08-11 | +| Unicode `use`/`extern crate` alias | `tests/import-extraction-unicode-identifiers.test.ts` | `use ... as alias` and `extern crate ... as alias` accept full Unicode XID_Start/XID_Continue identifiers. | Internal regression test | 2026-08-16 | ## SCSS diff --git a/src/graphs/specifiers.ts b/src/graphs/specifiers.ts index 41e5cca8..e7017172 100644 --- a/src/graphs/specifiers.ts +++ b/src/graphs/specifiers.ts @@ -28,6 +28,7 @@ import { isGraphOnlyLanguage, } from "../documentLinks.js"; import { sliceText, unquote } from "../util/ast.js"; +import { PYTHON_IDENTIFIER_SOURCE } from "../util/identifiers.js"; import { isRustCfgTestStatement, utf8ByteOffsetToStringIndex } from "../util/rustTestModules.js"; import { extractJsTsSpecifiers, extractPythonSpecifiers, type ModuleSpecifier } from "../util/specifiers.js"; @@ -228,6 +229,18 @@ function extractCssModuleSpecifiers(source: string): ModuleSpecifier[] { return out; } +// Python module/package names are dotted sequences of PEP 3131 Unicode identifiers; a +// per-segment character class (rather than Unicode letters/digits spanning the dots) keeps +// a digit from matching directly after a `.` separator. +const PYTHON_NATIVE_IMPORT_SPEC_PATTERN = new RegExp( + String.raw`^(${PYTHON_IDENTIFIER_SOURCE}(?:\.${PYTHON_IDENTIFIER_SOURCE})*)(?:\s+as\s+${PYTHON_IDENTIFIER_SOURCE})?$`, + "u", +); +const PYTHON_NATIVE_FROM_PATTERN = new RegExp( + String.raw`^\s*from\s+(\.*)(${PYTHON_IDENTIFIER_SOURCE}(?:\.${PYTHON_IDENTIFIER_SOURCE})*)?\s+import\b`, + "u", +); + export function collectModuleSpecifiersFromSource( support: LanguageSupport, _lang: unknown, @@ -275,13 +288,12 @@ export function collectModuleSpecifiersFromSource( .map((entry) => entry.trim()) .filter(Boolean); for (const spec of list) { - // Python module/package names permit Unicode identifiers (PEP 3131). - const parsed = spec.match(/^([\p{L}_][\p{L}\p{N}_.]*)(?:\s+as\s+[\p{L}_][\p{L}\p{N}_]*)?$/u); + const parsed = spec.match(PYTHON_NATIVE_IMPORT_SPEC_PATTERN); if (parsed?.[1]) out.push({ spec: parsed[1] }); } continue; } - const mFrom = /^\s*from\s+(\.*)([\p{L}_][\p{L}\p{N}_.]*)?\s+import\b/u.exec(stmtText); + const mFrom = PYTHON_NATIVE_FROM_PATTERN.exec(stmtText); if (mFrom) { const dots = mFrom[1] ?? ""; const name = mFrom[2] ?? ""; diff --git a/src/languages/importStatementParsers.ts b/src/languages/importStatementParsers.ts index 9183a32b..727c5315 100644 --- a/src/languages/importStatementParsers.ts +++ b/src/languages/importStatementParsers.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { XID_IDENTIFIER_SOURCE } from "../util/identifiers.js"; +import { PHP_IDENTIFIER_SOURCE, XID_IDENTIFIER_SOURCE } from "../util/identifiers.js"; import { isAbsoluteFilePath, normalizePath } from "../util/paths.js"; export type ParsedRustImportStatement = @@ -109,6 +109,8 @@ export type ParsedPhpImportStatement = export type PhpImportType = "class" | "function" | "const"; +const PHP_USE_ALIAS_PATTERN = new RegExp(String.raw`^(.*?)\s+as\s+(${PHP_IDENTIFIER_SOURCE})$`, "iu"); + function splitTopLevelCommaList(input: string): string[] { const items: string[] = []; let depth = 0; @@ -150,8 +152,7 @@ function parsePhpImportClause(rawClause: string, importType: PhpImportType): Par memberType = "const"; } const body = (typedMemberMatch?.[2] ?? member).trim(); - // PHP identifiers permit any byte >= 0x80, which in practice means non-ASCII UTF-8. - const aliasMatch = body.match(/^(.*?)\s+as\s+([\p{L}_][\p{L}\p{N}_]*)$/iu); + const aliasMatch = body.match(PHP_USE_ALIAS_PATTERN); const fullPath = `${prefix}${(aliasMatch?.[1] ?? body).trim()}`; const parts = fullPath.split("\\").filter(Boolean); const imported = parts[parts.length - 1]; @@ -168,7 +169,7 @@ function parsePhpImportClause(rawClause: string, importType: PhpImportType): Par return results; } - const aliasMatch = clause.match(/^(.*?)\s+as\s+([\p{L}_][\p{L}\p{N}_]*)$/iu); + const aliasMatch = clause.match(PHP_USE_ALIAS_PATTERN); const fullPath = (aliasMatch?.[1] ?? clause).trim(); const parts = fullPath.split("\\").filter(Boolean); const imported = parts[parts.length - 1]; diff --git a/src/util/git.ts b/src/util/git.ts index cae171e9..5d4b1dcf 100644 --- a/src/util/git.ts +++ b/src/util/git.ts @@ -1,4 +1,5 @@ import { spawn, type ChildProcess } from "node:child_process"; +import { StringDecoder } from "node:string_decoder"; import path from "node:path"; import { stringifyUnknown } from "./ast.js"; import { normalizePath } from "./paths.js"; @@ -16,6 +17,17 @@ const MAX_GIT_HASH_OBJECT_ARGUMENT_BYTES = 24 * 1024; let gitExecutableForTests: string | null = null; +/** Git's C-style path quoting single-character escapes for otherwise-unrepresentable control bytes. */ +const GIT_QUOTED_PATH_SINGLE_BYTE_ESCAPES: Record = { + a: 0x07, + b: 0x08, + f: 0x0c, + n: 0x0a, + r: 0x0d, + t: 0x09, + v: 0x0b, +}; + /** Decodes Git's optional C-style quoted pathname representation without trimming legal path bytes. */ export function decodeGitPath(rawPath: string): string { if (!rawPath.startsWith('"') || !rawPath.endsWith('"')) { @@ -44,18 +56,9 @@ export function decodeGitPath(rawPath: string): string { index += 2; continue; } - if (next === "n") { - bytes.push(0x0a); - index += 2; - continue; - } - if (next === "r") { - bytes.push(0x0d); - index += 2; - continue; - } - if (next === "t") { - bytes.push(0x09); + const singleByteEscape = GIT_QUOTED_PATH_SINGLE_BYTE_ESCAPES[next ?? ""]; + if (singleByteEscape !== undefined) { + bytes.push(singleByteEscape); index += 2; continue; } @@ -171,9 +174,16 @@ export async function runGit( return; } + // Decode incrementally per stream: a naive `chunk.toString()` on each independent + // Buffer can split a multibyte UTF-8 sequence across chunk boundaries, replacing both + // halves with U+FFFD. StringDecoder buffers a dangling partial sequence until the next + // chunk completes it. + const stdoutDecoder = new StringDecoder("utf8"); + const stderrDecoder = new StringDecoder("utf8"); + stdoutStream.on("data", (chunk: Buffer | string) => { - const textChunk = typeof chunk === "string" ? chunk : chunk.toString(); - totalBytes += Buffer.byteLength(textChunk, "utf8"); + const chunkBytes = typeof chunk === "string" ? Buffer.byteLength(chunk, "utf8") : chunk.length; + totalBytes += chunkBytes; if (totalBytes > maxBuffer) { killGitChild(child); settle(() => @@ -181,15 +191,17 @@ export async function runGit( ); return; } - stdout += textChunk; + stdout += typeof chunk === "string" ? chunk : stdoutDecoder.write(chunk); }); stderrStream.on("data", (chunk: Buffer | string) => { - stderr += typeof chunk === "string" ? chunk : chunk.toString(); + stderr += typeof chunk === "string" ? chunk : stderrDecoder.write(chunk); }); child.on("error", (error) => { settle(() => reject(createGitError(projectRoot, args, error))); }); child.on("close", (code, signalName) => { + stdout += stdoutDecoder.end(); + stderr += stderrDecoder.end(); settle(() => { if (timedOut) { reject( diff --git a/src/util/identifiers.ts b/src/util/identifiers.ts index 65904c64..34ba4e2d 100644 --- a/src/util/identifiers.ts +++ b/src/util/identifiers.ts @@ -6,3 +6,10 @@ export const XID_IDENTIFIER_SOURCE = String.raw`[_\p{XID_Start}][_\p{XID_Continu /** Python identifiers use normalized Unicode XID properties (PEP 3131). */ export const PYTHON_IDENTIFIER_SOURCE = XID_IDENTIFIER_SOURCE; + +/** + * PHP identifiers permit ASCII letters/underscore or any byte from 0x80-0xff at every + * position (non-ASCII bytes are unrestricted), with ASCII digits allowed only after the + * first character. + */ +export const PHP_IDENTIFIER_SOURCE = String.raw`[A-Za-z_\u{80}-\u{10FFFF}][A-Za-z0-9_\u{80}-\u{10FFFF}]*`; diff --git a/src/util/specifiers.ts b/src/util/specifiers.ts index ed272c47..c7f7d35d 100644 --- a/src/util/specifiers.ts +++ b/src/util/specifiers.ts @@ -1,5 +1,6 @@ import path from "node:path"; import { buildJsLikeLiteralMask, stripJsLikeComments, stripPythonCommentsAndStrings } from "./comments.js"; +import { PYTHON_IDENTIFIER_SOURCE } from "./identifiers.js"; import { normalizePath } from "./paths.js"; export type ModuleSpecifierResolutionKind = "document" | "source" | "stylesheet"; @@ -248,14 +249,21 @@ export function extractJsTsDynamicSpecifiers(source: string, fromFile: string, p return out; } +// Python module/package names are dotted sequences of PEP 3131 Unicode identifiers; a +// per-segment character class (rather than Unicode letters/digits spanning the dots) keeps +// a digit from matching directly after a `.` separator. +const PYTHON_DOTTED_NAME_SOURCE = String.raw`${PYTHON_IDENTIFIER_SOURCE}(?:\.${PYTHON_IDENTIFIER_SOURCE})*`; + export function extractPythonSpecifiers(source: string): string[] { const out: string[] = []; try { const cleaned = stripPythonCommentsAndStrings(source); - // Python module/package names permit Unicode identifiers (PEP 3131). - const reImport = /^\s*import\s+([\p{L}_][\p{L}\p{N}_.]*)/gmu; + const reImport = new RegExp(String.raw`^\s*import\s+(${PYTHON_DOTTED_NAME_SOURCE})`, "gmu"); for (const match of cleaned.matchAll(reImport)) out.push(match[1]!); - const reFrom = /^\s*from\s+(\.+(?:[\p{L}_][\p{L}\p{N}_.]*)?|[\p{L}_][\p{L}\p{N}_.]*)\s+import/gmu; + const reFrom = new RegExp( + String.raw`^\s*from\s+(\.+(?:${PYTHON_DOTTED_NAME_SOURCE})?|${PYTHON_DOTTED_NAME_SOURCE})\s+import`, + "gmu", + ); for (const match of cleaned.matchAll(reFrom)) out.push(match[1]!); } catch { /* parse fallback: ignore */ diff --git a/tests/git-diff-semantics.test.ts b/tests/git-diff-semantics.test.ts index 0ff31e59..2e74a158 100644 --- a/tests/git-diff-semantics.test.ts +++ b/tests/git-diff-semantics.test.ts @@ -1,9 +1,9 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, afterEach } from "vitest"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { listChangedFiles, listUntrackedFiles, getUnifiedDiff } from "../src/util.js"; -import { decodeGitPath } from "../src/util/git.js"; +import { decodeGitPath, runGit, setGitExecutableForTests } from "../src/util/git.js"; import { parseUnifiedDiff } from "../src/impact/parse.js"; import { runGit as git } from "./helpers/git.js"; @@ -262,6 +262,7 @@ describe("Git C-style quoted path decoding", () => { ['"emoji \\360\\237\\230\\200.ts"', "emoji 😀.ts"], ['"quote\\" and slash\\\\.ts"', 'quote" and slash\\.ts'], ['"tab\\tline\\ncarriage\\r.ts"', "tab\tline\ncarriage\r.ts"], + ['"bell\\avtab\\vformfeed\\fbackspace\\b.ts"', "bell\x07vtab\x0bformfeed\x0cbackspace\x08.ts"], ['"\\1\\12\\123"', "\x01\nS"], ['"unknown\\qtrailing\\"', "unknown\\qtrailing\\"], ]; @@ -272,6 +273,39 @@ describe("Git C-style quoted path decoding", () => { }); }); +describe("git subprocess stdout decoding across chunk boundaries", () => { + afterEach(() => { + setGitExecutableForTests(null); + }); + + it("reassembles a multibyte UTF-8 sequence split across separate stdout writes", async () => { + const root = await makeGitTempDir("codegraph-git-chunk-split-"); + try { + // The emoji U+1F600 encodes as 4 UTF-8 bytes (F0 9F 98 80); writing the first two + // bytes, then yielding a macrotask before writing the rest, forces two independent + // stdout "data" events. Decoding each chunk independently (the previous + // `chunk.toString()` behavior) would replace both halves with U+FFFD instead of + // reassembling the character. The 20ms delay runs inside the spawned child process + // (a separate OS process/JS realm from this test), not this test file, so Vitest fake + // timers cannot control it; a real, short delay is the only way to force two distinct + // pipe writes. + const script = [ + "process.stdout.write(Buffer.from([0x41, 0xf0, 0x9f]));", + "setTimeout(() => {", + " process.stdout.write(Buffer.from([0x98, 0x80, 0x42]));", + " process.exit(0);", + "}, 20);", + ].join("\n"); + + setGitExecutableForTests(process.execPath); + const { stdout } = await runGit(root, ["-e", script]); + expect(stdout).toBe("A\u{1f600}B"); + } finally { + await removeGitTempDir(root); + } + }); +}); + describe("listUntrackedFiles", () => { it("lists new files Git has not been told to track", async () => { const root = await makeGitTempDir("codegraph-git-untracked-"); diff --git a/tests/import-extraction-unicode-identifiers.test.ts b/tests/import-extraction-unicode-identifiers.test.ts index 4eada37b..c1cdb3ca 100644 --- a/tests/import-extraction-unicode-identifiers.test.ts +++ b/tests/import-extraction-unicode-identifiers.test.ts @@ -10,6 +10,8 @@ import { parseRustImportStatement, } from "../src/languages/importStatementParsers.js"; import { extractJsTsSpecifiers, extractPythonSpecifiers } from "../src/util.js"; +import { collectModuleSpecifiersFromSource } from "../src/graphs.js"; +import { supportById } from "../src/languages.js"; import { buildProjectIndex } from "../src/index.js"; import { collectJsTextImports } from "../src/indexer/imports/jsTextImports.js"; import { collectNativeCaptureImportBindings } from "../src/indexer/imports/nativeCaptures.js"; @@ -55,6 +57,15 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { importType: "class", }, ]); + // PHP permits any byte >= 0x80 in an identifier, not just Unicode letters/digits + // (\p{L}/\p{N}); an emoji alias is valid PHP even though it is outside \p{L}. + expect(parsePhpImportStatement("use App\\Foo as \u{1f600};")).toEqual([ + expect.objectContaining({ local: "\u{1f600}" }), + ]); + expect(parsePhpImportStatement("use App\\{Foo as \u{1f600}, Bar};")).toEqual([ + expect.objectContaining({ imported: "Foo", local: "\u{1f600}" }), + expect.objectContaining({ imported: "Bar", local: "Bar" }), + ]); }); it("Kotlin: import alias", () => { @@ -85,6 +96,14 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { it("Python fallback module-specifier extraction: import/from with Unicode module names", () => { expect(extractPythonSpecifiers("import créer\n")).toEqual(["créer"]); expect(extractPythonSpecifiers("from créer import x\n")).toContain("créer"); + // PEP 3131 XID_Continue includes combining marks; a per-code-point \p{L}/\p{N} class + // stops before the trailing combining acute accent, silently dropping it from the + // captured module name. + expect(extractPythonSpecifiers("import café\u0301\n")).toEqual(["café\u0301"]); + // A dotted segment must itself start with an identifier character: matching the whole + // continuation class (letters/digits/dots) across the separator let a digit immediately + // follow a `.`, which Python's grammar never allows. + expect(extractPythonSpecifiers("import pkg.2mod\n")).toEqual(["pkg"]); }); it("Python import bindings accept combining-mark continuations", async () => { @@ -278,4 +297,21 @@ describe("Unicode import parser seams", () => { { spec: "package", exportCondition: "require" }, ]); }); + + it("parses Unicode Python module names from native-query statement captures", () => { + const support = supportById("python")!; + // Mirrors the shape collectModuleSpecifiersFromSource reads from a native compact + // imports execution: one match per statement, with the full statement text under a + // "stmt" capture. + const specs = collectModuleSpecifiersFromSource(support, undefined, "import café\u0301\nfrom pkg import x\n", { + compactNativeImports: { + imports: [ + { patternIndex: 0, captures: [{ name: "stmt", text: "import café\u0301" }] }, + { patternIndex: 0, captures: [{ name: "stmt", text: "from pkg import x" }] }, + ], + }, + }); + + expect(specs).toEqual([{ spec: "café\u0301" }, { spec: "pkg" }]); + }); }); From 13e019001ea9751cbac3b478772d7a8b4350583b Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 18:00:52 -0400 Subject: [PATCH 22/29] fix: address second review batch (Java/C# identifiers, byte-index reuse, doc formatting) - Java import parsing (native + text fallback) now uses a shared JAVA_IDENTIFIER_SOURCE covering JLS JavaLetter/JavaLetterOrDigit (Unicode letters, $, connecting punctuation, currency symbols, combining marks), fixing dropped bindings like 'import com.example.$Widget;'. - C# using-directive parsing (alias, static, plain) now uses a shared CSHARP_IDENTIFIER_SOURCE covering the ECMA-334 identifier grammar plus the optional '@' verbatim-identifier prefix (e.g. '@class'). - ProjectedSyntaxTree exposes its byteIndexMap publicly; locals-and-exports's ensureByteIndexMap() now reuses it via ensureTree() instead of rescanning the source a second time for every non-ASCII native-query file. - Fix a markdown-emphasis corruption in language-parity.md and scenario-catalog.md where bare 'ID_Start'/'XID_Start' underscores were reinterpreted as emphasis markers by the formatter; every identifier term is now backtick-wrapped. --- docs/coverage/js.md | 4 +- docs/language-parity.md | 2 +- docs/scenario-catalog.md | 16 ++++---- src/indexer/imports/languageSpecific.ts | 7 +++- src/indexer/locals-and-exports.ts | 7 +++- src/languages/importStatementParsers.ts | 38 ++++++++++++++----- src/native/projectedTree.ts | 9 +++-- src/util/identifiers.ts | 15 ++++++++ ...ort-extraction-unicode-identifiers.test.ts | 15 ++++++++ tests/native-query-results.test.ts | 32 ++++++++++++++-- 10 files changed, 115 insertions(+), 30 deletions(-) diff --git a/docs/coverage/js.md b/docs/coverage/js.md index 56955163..f868f128 100644 --- a/docs/coverage/js.md +++ b/docs/coverage/js.md @@ -6,9 +6,9 @@ Source: `coverage/js/lcov.info` | Metric | Hit | Found | Coverage | | --------- | ----: | ----: | -------: | -| Lines | 27458 | 30231 | 90.83% | +| Lines | 27468 | 30241 | 90.83% | | Functions | 4563 | 4842 | 94.24% | -| Branches | 20693 | 26098 | 79.29% | +| Branches | 20694 | 26100 | 79.29% | ## Least-covered Files diff --git a/docs/language-parity.md b/docs/language-parity.md index 74d42302..53ffcb9c 100644 --- a/docs/language-parity.md +++ b/docs/language-parity.md @@ -92,7 +92,7 @@ Notes: - JavaScript expands `module.exports = { ...source }` for statically resolvable CommonJS imports and local object literals. Dynamic spread sources remain explicit namespace-reexport markers rather than silently disappearing. - Node `package.json#exports` condition matching follows author key order with mutually exclusive `import`/`require` modes threaded from ESM `import`/`import()` versus CommonJS `require()` (and TypeScript `import x = require(...)`). Nested conditions, array fallbacks, and `default` termination match Node for those cases. Custom `--conditions`, `browser`/`types`/`development`/`production`, import attributes, and `#imports` maps are not modeled. - Ruby treats `Constant = Struct.new(...)` as a class-kind symbol and a synthetic detailed class declaration. Runtime-computed class factories remain outside this recognition. -- Import and alias binding extraction accepts each source language's real identifier grammar rather than an ASCII-narrowed approximation: JS/TS/TSX use ID*Start/ID_Continue plus `$`/`*`and ZWNJ/ZWJ continuations, Python uses PEP 3131 XID_Start/XID_Continue, Rust/Go/Java/Kotlin use their Unicode letter/XID identifier rules, and PHP accepts any byte`>= 0x80` at any identifier position. This covers native-query statement parsing, text/regex fallback recovery, and specifier extraction for those languages. +- Import and alias binding extraction accepts each source language's real identifier grammar rather than an ASCII-narrowed approximation: JS/TS/TSX use `ID_Start`/`ID_Continue` plus `$` and `_` (and ZWNJ/ZWJ continuations), Python uses PEP 3131 `XID_Start`/`XID_Continue`, Rust/Go/Java/Kotlin use their Unicode letter/XID identifier rules, and PHP accepts any byte `>= 0x80` at any identifier position. This covers native-query statement parsing, text/regex fallback recovery, and specifier extraction for those languages. ## Project file discovery coverage diff --git a/docs/scenario-catalog.md b/docs/scenario-catalog.md index 49484416..6a5e3461 100644 --- a/docs/scenario-catalog.md +++ b/docs/scenario-catalog.md @@ -185,7 +185,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Class field navigation | `tests/class-field-locals.test.ts` | Public and `#private` class field declarations are indexed as navigable variable definitions; `goto` resolves constructed-receiver field access (`const w = new Widget(); w.size`) to the field declaration. | Internal regression test | 2026-08-09 | | CommonJS spread exports | `tests/samples/language-regressions/javascript/*.js`, `tests/languages/javascript.test.ts` | `module.exports = { ...base, ...local }` reexports static `require()` and local-object members; a dynamic spread retains an explicit uncertainty marker. | Internal regression fixture | 2026-08-11 | | Conditional package exports order | `tests/package-exports.test.ts`, `tests/node-resolution.test.ts` | `exports` conditions evaluate in author key order (`node` before `import` flips with key order); `require()` consumers resolve the `require` target and ESM `import` consumers resolve the `import` target for dual `.cjs`/`.mjs` packages, including nested conditions and `default` termination. | Internal regression test | 2026-08-11 | -| Unicode import identifiers | `tests/import-extraction-unicode-identifiers.test.ts` | CommonJS destructuring, `import =`/`require()` equals bindings, and text-fallback import/alias extraction accept full ID*Start/ID_Continue identifiers (including `$`/`*` and ZWNJ/ZWJ continuations), not just ASCII/letter-digit subsets. | Internal regression test | 2026-08-16 | +| Unicode import identifiers | `tests/import-extraction-unicode-identifiers.test.ts` | CommonJS destructuring, `import =`/`require()` equals bindings, and text-fallback import/alias extraction accept full `ID_Start`/`ID_Continue` identifiers (including `$` and `_`, plus ZWNJ/ZWJ continuations), not just ASCII/letter-digit subsets. | Internal regression test | 2026-08-16 | ## Kotlin @@ -207,12 +207,12 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. ## Python -| Scenario | Sample | Expected behavior | Source | Date added | -| ------------------------------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------- | ---------- | -| Relative `from` imports | `tests/samples/python/relative-imports.py` | Dependency graph includes edges to `utils.py` and `helpers.py` for relative `from` imports. | https://github.com/tree-sitter/tree-sitter-python | 2026-01-22 | -| `__all__` export filtering | `tests/languages/python.test.ts` | Export extraction respects `__all__` tuple/list assignments and avoids false positives from nearby strings. | Internal regression test | 2026-03-22 | -| Match bindings and `.pyi` stubs | `tests/samples/language-regressions/python/*`, `tests/languages/python.test.ts` | Tuple and `as` pattern captures are navigable locals with references, and `.pyi` files are discovered and their class/function symbols are indexed. | Internal regression fixture | 2026-08-11 | -| Unicode module names | `tests/import-extraction-unicode-identifiers.test.ts` | `import`/`from` module and alias names accept full PEP 3131 XID_Start/XID_Continue identifiers, including combining-mark continuations, in native-query parsing and text-fallback specifier extraction. A dotted segment must itself start with an identifier character, so a digit cannot immediately follow a `.` separator. | Internal regression test | 2026-08-16 | +| Scenario | Sample | Expected behavior | Source | Date added | +| ------------------------------- | ------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | ---------- | +| Relative `from` imports | `tests/samples/python/relative-imports.py` | Dependency graph includes edges to `utils.py` and `helpers.py` for relative `from` imports. | https://github.com/tree-sitter/tree-sitter-python | 2026-01-22 | +| `__all__` export filtering | `tests/languages/python.test.ts` | Export extraction respects `__all__` tuple/list assignments and avoids false positives from nearby strings. | Internal regression test | 2026-03-22 | +| Match bindings and `.pyi` stubs | `tests/samples/language-regressions/python/*`, `tests/languages/python.test.ts` | Tuple and `as` pattern captures are navigable locals with references, and `.pyi` files are discovered and their class/function symbols are indexed. | Internal regression fixture | 2026-08-11 | +| Unicode module names | `tests/import-extraction-unicode-identifiers.test.ts` | `import`/`from` module and alias names accept full PEP 3131 `XID_Start`/`XID_Continue` identifiers, including combining-mark continuations, in native-query parsing and text-fallback specifier extraction. A dotted segment must itself start with an identifier character, so a digit cannot immediately follow a `.` separator. | Internal regression test | 2026-08-16 | ## PHP @@ -251,7 +251,7 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. | Reexports and nested modules | `tests/samples/rust/reexports.rs`, `tests/samples/rust/nested.rs`, `tests/samples/rust/nested_service.rs` | Go-to-definition and references cover nested-module type resolution, and native semantic coverage keeps those nested/reexport fixtures stable. | Internal regression fixture | 2026-03-23 | | Traits, impls, enums, and exported types | `tests/samples/rust/models.rs` | Symbol extraction includes trait declarations, enum declarations, enum variants, impl-backed methods, and exported structs from Rust modules. | Internal regression fixture | 2026-03-22 | | `macro_rules!` definitions | `tests/samples/rust/.regressions/macros.rs`, `tests/languages/rust.test.ts` | Macro definitions are chunked and indexed; macro invocations resolve to the definition and count as references. | Internal regression fixture | 2026-08-11 | -| Unicode `use`/`extern crate` alias | `tests/import-extraction-unicode-identifiers.test.ts` | `use ... as alias` and `extern crate ... as alias` accept full Unicode XID_Start/XID_Continue identifiers. | Internal regression test | 2026-08-16 | +| Unicode `use`/`extern crate` alias | `tests/import-extraction-unicode-identifiers.test.ts` | `use ... as alias` and `extern crate ... as alias` accept full Unicode `XID_Start`/`XID_Continue` identifiers. | Internal regression test | 2026-08-16 | ## SCSS diff --git a/src/indexer/imports/languageSpecific.ts b/src/indexer/imports/languageSpecific.ts index d1ec223b..675b52a8 100644 --- a/src/indexer/imports/languageSpecific.ts +++ b/src/indexer/imports/languageSpecific.ts @@ -1,5 +1,6 @@ import path from "node:path"; import { + JAVA_DOTTED_NAME_SOURCE, parseCsharpUsingDirective, parseJavaImportStatement, parseKotlinImportStatement, @@ -86,8 +87,10 @@ async function appendJavaTextImports(context: LanguageSpecificImportContext): Pr if (context.languageId !== "java" || context.getBindings().length) { return; } - // Java identifiers permit Unicode letters, not just ASCII. - const importPattern = /^\s*import\s+(static\s+)?([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)\s*;/gmu; + const importPattern = new RegExp( + String.raw`^\s*import\s+(static\s+)?(${JAVA_DOTTED_NAME_SOURCE}(?:\.\*)?)\s*;`, + "gmu", + ); for (const match of context.source.matchAll(importPattern)) { const isStatic = !!match[1]; const rawSpec = match[2]; diff --git a/src/indexer/locals-and-exports.ts b/src/indexer/locals-and-exports.ts index 6ddbce89..3ed2c0ca 100644 --- a/src/indexer/locals-and-exports.ts +++ b/src/indexer/locals-and-exports.ts @@ -381,9 +381,14 @@ export function collectLocalsAndExportsFromSource( // Lazily build once: converts every native capture's UTF-8 byte offsets to UTF-16 // string indexes in O(1) per capture instead of rescanning the source per offset. + // `ensureTree()` builds the same map internally when native mode is active, so route + // through it first and reuse that map instead of scanning the source a second time. let byteIndexMap: ByteToStringIndexMap | null = null; const ensureByteIndexMap = (): ByteToStringIndexMap => { - if (!byteIndexMap) byteIndexMap = buildByteToStringIndexMap(source); + if (byteIndexMap) return byteIndexMap; + const enrichmentTree = ensureTree(); + byteIndexMap = + enrichmentTree instanceof ProjectedSyntaxTree ? enrichmentTree.byteIndexMap : buildByteToStringIndexMap(source); return byteIndexMap; }; diff --git a/src/languages/importStatementParsers.ts b/src/languages/importStatementParsers.ts index 727c5315..1ea1057c 100644 --- a/src/languages/importStatementParsers.ts +++ b/src/languages/importStatementParsers.ts @@ -1,5 +1,10 @@ import path from "node:path"; -import { PHP_IDENTIFIER_SOURCE, XID_IDENTIFIER_SOURCE } from "../util/identifiers.js"; +import { + CSHARP_IDENTIFIER_SOURCE, + JAVA_IDENTIFIER_SOURCE, + PHP_IDENTIFIER_SOURCE, + XID_IDENTIFIER_SOURCE, +} from "../util/identifiers.js"; import { isAbsoluteFilePath, normalizePath } from "../util/paths.js"; export type ParsedRustImportStatement = @@ -397,6 +402,25 @@ export function parseKotlinImportStatement(stmtText: string): ParsedKotlinImport }; } +export const JAVA_DOTTED_NAME_SOURCE = String.raw`${JAVA_IDENTIFIER_SOURCE}(?:\.${JAVA_IDENTIFIER_SOURCE})*`; +const JAVA_IMPORT_PATTERN = new RegExp( + String.raw`^\s*import\s+(static\s+)?(${JAVA_DOTTED_NAME_SOURCE}(?:\.\*)?)\s*;?\s*$`, + "u", +); + +const CSHARP_DOTTED_NAME_SOURCE = String.raw`${CSHARP_IDENTIFIER_SOURCE}(?:\.${CSHARP_IDENTIFIER_SOURCE})*`; +const CSHARP_USING_ALIAS_PATTERN = new RegExp( + String.raw`^(?:global\s+)?using\s+(${CSHARP_IDENTIFIER_SOURCE})\s*=\s*(${CSHARP_DOTTED_NAME_SOURCE})\s*;?$`, + "u", +); +const CSHARP_USING_STATIC_PATTERN = new RegExp( + String.raw`^(?:global\s+)?using\s+static\s+(${CSHARP_DOTTED_NAME_SOURCE})\s*;?$`, + "u", +); +const CSHARP_USING_PLAIN_PATTERN = new RegExp( + String.raw`^(?:global\s+)?using\s+(${CSHARP_DOTTED_NAME_SOURCE})\s*;?$`, + "u", +); export type ParsedJavaImportStatement = | { kind: "named"; @@ -411,8 +435,7 @@ export type ParsedJavaImportStatement = }; export function parseJavaImportStatement(stmtText: string): ParsedJavaImportStatement | null { - // Java identifiers permit Unicode letters, not just ASCII. - const match = stmtText.trim().match(/^\s*import\s+(static\s+)?([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)\s*;?\s*$/u); + const match = stmtText.trim().match(JAVA_IMPORT_PATTERN); const rawSpec = match?.[2]; if (!rawSpec) return null; const isStatic = !!match?.[1]; @@ -438,10 +461,7 @@ export function parseJavaImportStatement(stmtText: string): ParsedJavaImportStat export function parseCsharpUsingDirective(stmtText: string): ParsedCsharpUsingDirective | null { const trimmed = stmtText.trim(); - // C# identifiers permit Unicode letter categories, not just ASCII. - const aliasMatch = trimmed.match( - /^(?:global\s+)?using\s+([\p{L}_][\p{L}\p{N}_]*)\s*=\s*([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u, - ); + const aliasMatch = trimmed.match(CSHARP_USING_ALIAS_PATTERN); if (aliasMatch?.[1] && aliasMatch[2]) { return { from: aliasMatch[2], @@ -450,7 +470,7 @@ export function parseCsharpUsingDirective(stmtText: string): ParsedCsharpUsingDi }; } - const staticMatch = trimmed.match(/^(?:global\s+)?using\s+static\s+([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u); + const staticMatch = trimmed.match(CSHARP_USING_STATIC_PATTERN); if (staticMatch?.[1]) { return { from: staticMatch[1], @@ -458,7 +478,7 @@ export function parseCsharpUsingDirective(stmtText: string): ParsedCsharpUsingDi }; } - const plainMatch = trimmed.match(/^(?:global\s+)?using\s+([\p{L}_][\p{L}\p{N}_.]*)\s*;?$/u); + const plainMatch = trimmed.match(CSHARP_USING_PLAIN_PATTERN); if (!plainMatch?.[1]) return null; return { from: plainMatch[1], diff --git a/src/native/projectedTree.ts b/src/native/projectedTree.ts index a26cdf5a..5e8128e6 100644 --- a/src/native/projectedTree.ts +++ b/src/native/projectedTree.ts @@ -14,12 +14,13 @@ export type ProjectedPosition = { export class ProjectedSyntaxTree { readonly source: string; private readonly nodesById: Map; - private readonly byteMap: ByteToStringIndexMap; + /** Shared byte-offset -> UTF-16 string-index map; reuse this instead of rebuilding one. */ + readonly byteIndexMap: ByteToStringIndexMap; readonly rootNode: ProjectedSyntaxNode; constructor(source: string, tree: NativeSyntaxTree) { this.source = source; - this.byteMap = buildByteToStringIndexMap(source); + this.byteIndexMap = buildByteToStringIndexMap(source); this.nodesById = new Map(); for (const node of tree.nodes) { this.nodesById.set(node.id, new ProjectedSyntaxNode(this, node)); @@ -36,11 +37,11 @@ export class ProjectedSyntaxTree { } stringIndexForByte(byteIndex: number): number { - return stringIndexForByte(this.byteMap, byteIndex); + return stringIndexForByte(this.byteIndexMap, byteIndex); } positionForPoint(point: NativePoint): ProjectedPosition { - return stringPositionForBytePoint(this.byteMap, point); + return stringPositionForBytePoint(this.byteIndexMap, point); } } diff --git a/src/util/identifiers.ts b/src/util/identifiers.ts index 34ba4e2d..3d58fc20 100644 --- a/src/util/identifiers.ts +++ b/src/util/identifiers.ts @@ -13,3 +13,18 @@ export const PYTHON_IDENTIFIER_SOURCE = XID_IDENTIFIER_SOURCE; * first character. */ export const PHP_IDENTIFIER_SOURCE = String.raw`[A-Za-z_\u{80}-\u{10FFFF}][A-Za-z0-9_\u{80}-\u{10FFFF}]*`; + +/** + * Java identifiers (JLS `JavaLetter`/`JavaLetterOrDigit`) permit Unicode letters, `$`, + * connecting-punctuation characters (e.g. `_`), and currency symbols at the first position; + * continuation additionally allows Unicode digits and combining marks. + */ +export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Sc}\p{Pc}$][\p{L}\p{N}\p{Sc}\p{Pc}\p{Mn}\p{Mc}$]*`; + +/** + * C# identifiers (ECMA-334 `identifier-start-character`/`identifier-part-character`) permit + * Unicode letter categories or an underscore at the first position, plus an optional leading + * `@` for a verbatim identifier (escaping a keyword, e.g. `@class`); continuation additionally + * allows decimal digits, connecting-punctuation, combining marks, and formatting characters. + */ +export const CSHARP_IDENTIFIER_SOURCE = String.raw`@?[\p{L}\p{Pc}_][\p{L}\p{N}\p{Pc}\p{Mn}\p{Mc}\p{Cf}_]*`; diff --git a/tests/import-extraction-unicode-identifiers.test.ts b/tests/import-extraction-unicode-identifiers.test.ts index c1cdb3ca..6d736bef 100644 --- a/tests/import-extraction-unicode-identifiers.test.ts +++ b/tests/import-extraction-unicode-identifiers.test.ts @@ -84,6 +84,14 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { imported: "Créer", isStatic: false, }); + // JLS JavaLetter includes `$` and connecting-punctuation characters at every position, + // not just Unicode letters/digits. + expect(parseJavaImportStatement("import com.example.$Widget;")).toEqual({ + kind: "named", + from: "com.example.$Widget", + imported: "$Widget", + isStatic: false, + }); }); it("C#: using alias to a Unicode-named alias", () => { @@ -92,6 +100,13 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { alias: "créer", isStatic: false, }); + // A verbatim identifier (`@` prefix) escapes a reserved keyword; `@class` is a legal + // C# identifier distinct from the `class` keyword. + expect(parseCsharpUsingDirective("using @class = Some.@class;")).toEqual({ + from: "Some.@class", + alias: "@class", + isStatic: false, + }); }); it("Python fallback module-specifier extraction: import/from with Unicode module names", () => { expect(extractPythonSpecifiers("import créer\n")).toEqual(["créer"]); diff --git a/tests/native-query-results.test.ts b/tests/native-query-results.test.ts index 9bfa02ac..666c5579 100644 --- a/tests/native-query-results.test.ts +++ b/tests/native-query-results.test.ts @@ -1,6 +1,9 @@ -import { describe, expect, it } from "vitest"; -import { buildByteToStringIndexMap } from "../src/native/byteIndex.js"; +import { describe, expect, it, vi } from "vitest"; +import * as byteIndexModule from "../src/native/byteIndex.js"; import { rangeFromNativeCapture } from "../src/native/queryResults.js"; +import { collectLocalsAndExportsFromSource } from "../src/indexer.js"; +import { supportForFile } from "../src/languages.js"; +import { isNativeTreeSitterAvailable } from "../src/native/treeSitterNative.js"; describe("rangeFromNativeCapture", () => { it("converts UTF-8 byte indexes and point columns to UTF-16 range boundaries", () => { @@ -22,7 +25,7 @@ describe("rangeFromNativeCapture", () => { start: { row: 1, column: startColumn, index: startByteIndex }, end: { row: 1, column: endColumn, index: endByteIndex }, }, - buildByteToStringIndexMap(source), + byteIndexModule.buildByteToStringIndexMap(source), ); expect(range).toEqual({ @@ -32,3 +35,26 @@ describe("rangeFromNativeCapture", () => { expect(source.slice(range.start.index, range.end.index)).toBe(text); }); }); + +describe.runIf(isNativeTreeSitterAvailable())("locals/exports byte-index map reuse", () => { + it("builds the byte-offset index map once and shares it with the projected syntax tree", () => { + const file = "consumer.ts"; + const support = supportForFile(file)!; + const source = "export const café = 1;\nexport function uséCafé() { return café; }\n"; + const buildSpy = vi.spyOn(byteIndexModule, "buildByteToStringIndexMap"); + + try { + const moduleIndex = collectLocalsAndExportsFromSource(file, source, support, support.language(file)); + // Every native-capture range conversion (locals, exports) and every tree lookup during + // this call must share one byte-index map instead of each rescanning the source. + expect(buildSpy).toHaveBeenCalledTimes(1); + const local = moduleIndex.locals.find((entry) => entry.localName === "uséCafé"); + expect(local?.range).toEqual({ + start: { line: 2, column: 17, index: 39 }, + end: { line: 2, column: 24, index: 46 }, + }); + } finally { + buildSpy.mockRestore(); + } + }); +}); From c65f9e5b18e173ac7e94171555fd321880704858 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 18:15:59 -0400 Subject: [PATCH 23/29] fix: correct Java/C# identifier Unicode categories, document C# in parity docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - JAVA_IDENTIFIER_SOURCE: match Character.isJavaIdentifierStart/Part exactly (letter, letter-number Nl, currency symbol Sc, connecting punctuation Pc at every position; add decimal-digit Nd and identifier-ignorable Cf to continuation; drop combining marks and non-decimal numbers, which Java's real grammar rejects). - CSHARP_IDENTIFIER_SOURCE: match ECMA-334 exactly (letter, letter-number Nl, or literal underscore at the start only — not every Pc character; add decimal-digit Nd, connecting-punctuation Pc, combining marks Mn/Mc, and formatting Cf to continuation). - Add Nl/No/Cf/Mn boundary-case regressions to the Java and C# parser tests per review request. - Document the new C# identifier-grammar behavior in language-parity.md and add a scenario-catalog.md C# row, closing the doc gap the PR's Unicode-identifier note previously left for C#. --- docs/coverage/js.md | 2 +- docs/language-parity.md | 2 +- docs/scenario-catalog.md | 15 +++++++------ src/util/identifiers.ts | 22 +++++++++++-------- ...ort-extraction-unicode-identifiers.test.ts | 22 +++++++++++++++++++ 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/docs/coverage/js.md b/docs/coverage/js.md index f868f128..6e802d14 100644 --- a/docs/coverage/js.md +++ b/docs/coverage/js.md @@ -8,7 +8,7 @@ Source: `coverage/js/lcov.info` | --------- | ----: | ----: | -------: | | Lines | 27468 | 30241 | 90.83% | | Functions | 4563 | 4842 | 94.24% | -| Branches | 20694 | 26100 | 79.29% | +| Branches | 20696 | 26100 | 79.30% | ## Least-covered Files diff --git a/docs/language-parity.md b/docs/language-parity.md index 53ffcb9c..f274b9a8 100644 --- a/docs/language-parity.md +++ b/docs/language-parity.md @@ -92,7 +92,7 @@ Notes: - JavaScript expands `module.exports = { ...source }` for statically resolvable CommonJS imports and local object literals. Dynamic spread sources remain explicit namespace-reexport markers rather than silently disappearing. - Node `package.json#exports` condition matching follows author key order with mutually exclusive `import`/`require` modes threaded from ESM `import`/`import()` versus CommonJS `require()` (and TypeScript `import x = require(...)`). Nested conditions, array fallbacks, and `default` termination match Node for those cases. Custom `--conditions`, `browser`/`types`/`development`/`production`, import attributes, and `#imports` maps are not modeled. - Ruby treats `Constant = Struct.new(...)` as a class-kind symbol and a synthetic detailed class declaration. Runtime-computed class factories remain outside this recognition. -- Import and alias binding extraction accepts each source language's real identifier grammar rather than an ASCII-narrowed approximation: JS/TS/TSX use `ID_Start`/`ID_Continue` plus `$` and `_` (and ZWNJ/ZWJ continuations), Python uses PEP 3131 `XID_Start`/`XID_Continue`, Rust/Go/Java/Kotlin use their Unicode letter/XID identifier rules, and PHP accepts any byte `>= 0x80` at any identifier position. This covers native-query statement parsing, text/regex fallback recovery, and specifier extraction for those languages. +- Import and alias binding extraction accepts each source language's real identifier grammar rather than an ASCII-narrowed approximation: JS/TS/TSX use `ID_Start`/`ID_Continue` plus `$` and `_` (and ZWNJ/ZWJ continuations), Python uses PEP 3131 `XID_Start`/`XID_Continue`, Rust/Go/Kotlin use their Unicode letter/XID identifier rules, Java follows `Character.isJavaIdentifierStart`/`isJavaIdentifierPart` (Unicode letters, letter-numbers, currency symbols, connecting punctuation, decimal digits, and identifier-ignorable formatting characters), C# follows the ECMA-334 `identifier-start-character`/`identifier-part-character` grammar including the `@` verbatim-identifier prefix, and PHP accepts any byte `>= 0x80` at any identifier position. This covers native-query statement parsing, text/regex fallback recovery, and specifier extraction for those languages. ## Project file discovery coverage diff --git a/docs/scenario-catalog.md b/docs/scenario-catalog.md index 6a5e3461..5240cecc 100644 --- a/docs/scenario-catalog.md +++ b/docs/scenario-catalog.md @@ -84,13 +84,14 @@ Minimal catalog of Tree-sitter scenarios with sample coverage. ## C# -| Scenario | Sample | Expected behavior | Source | Date added | -| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | ---------- | -| Using directives | `tests/samples/csharp/Main.cs` | Dependency graph includes one edge per resolved target for multi-binding `using` directives, including `Utils.cs` and `Helpers.cs`. | https://github.com/tree-sitter/tree-sitter-c-sharp | 2026-08-11 | -| Alias using graph edges | `tests/samples/csharp/AliasOnly.cs`, `tests/samples/csharp/NamespaceAlias.cs` | Dependency graph keeps alias-based `using` directives pointed at their target type/namespace instead of the alias token. Alias-only semantic navigation is intentionally not claimed yet. | Internal regression fixture | 2026-03-29 | -| Global `using` namespace, alias, and static forms | `tests/samples/csharp/GlobalUsings.cs`, `tests/samples/csharp/Shared.cs`, `tests/languages/csharp.test.ts` | Dependency graph and import bindings retain `global using System.Text;`, namespace imports, alias imports, and static imports without duplicate resolved edges; namespace-imported types navigate to the project declaration. | Internal regression fixture | 2026-08-11 | -| Nested types, interfaces, and enums | `tests/samples/csharp/AdvancedTypes.cs` | Symbol extraction includes interfaces, nested classes, enums, enum members, and member methods inside namespace-scoped fixtures. | Internal regression fixture | 2026-03-22 | -| `record`/`record struct` declarations | `tests/samples/csharp/RecordTypes.cs` | Symbol extraction indexes record declarations as class-kind symbols, and record `implements`/base-list interface conformance participates in type hierarchy the same as an ordinary class. | Internal regression fixture | 2026-08-09 | +| Scenario | Sample | Expected behavior | Source | Date added | +| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------- | ---------- | +| Using directives | `tests/samples/csharp/Main.cs` | Dependency graph includes one edge per resolved target for multi-binding `using` directives, including `Utils.cs` and `Helpers.cs`. | https://github.com/tree-sitter/tree-sitter-c-sharp | 2026-08-11 | +| Alias using graph edges | `tests/samples/csharp/AliasOnly.cs`, `tests/samples/csharp/NamespaceAlias.cs` | Dependency graph keeps alias-based `using` directives pointed at their target type/namespace instead of the alias token. Alias-only semantic navigation is intentionally not claimed yet. | Internal regression fixture | 2026-03-29 | +| Global `using` namespace, alias, and static forms | `tests/samples/csharp/GlobalUsings.cs`, `tests/samples/csharp/Shared.cs`, `tests/languages/csharp.test.ts` | Dependency graph and import bindings retain `global using System.Text;`, namespace imports, alias imports, and static imports without duplicate resolved edges; namespace-imported types navigate to the project declaration. | Internal regression fixture | 2026-08-11 | +| Nested types, interfaces, and enums | `tests/samples/csharp/AdvancedTypes.cs` | Symbol extraction includes interfaces, nested classes, enums, enum members, and member methods inside namespace-scoped fixtures. | Internal regression fixture | 2026-03-22 | +| `record`/`record struct` declarations | `tests/samples/csharp/RecordTypes.cs` | Symbol extraction indexes record declarations as class-kind symbols, and record `implements`/base-list interface conformance participates in type hierarchy the same as an ordinary class. | Internal regression fixture | 2026-08-09 | +| Unicode `using` alias | `tests/import-extraction-unicode-identifiers.test.ts` | `using alias = Namespace;` accepts the ECMA-334 identifier grammar (Unicode letters, letter-numbers, combining marks, connecting punctuation, and formatting characters), including an optional `@` verbatim-identifier prefix such as `@class`. | Internal regression test | 2026-08-16 | ## CSS diff --git a/src/util/identifiers.ts b/src/util/identifiers.ts index 3d58fc20..34f5dc24 100644 --- a/src/util/identifiers.ts +++ b/src/util/identifiers.ts @@ -15,16 +15,20 @@ export const PYTHON_IDENTIFIER_SOURCE = XID_IDENTIFIER_SOURCE; export const PHP_IDENTIFIER_SOURCE = String.raw`[A-Za-z_\u{80}-\u{10FFFF}][A-Za-z0-9_\u{80}-\u{10FFFF}]*`; /** - * Java identifiers (JLS `JavaLetter`/`JavaLetterOrDigit`) permit Unicode letters, `$`, - * connecting-punctuation characters (e.g. `_`), and currency symbols at the first position; - * continuation additionally allows Unicode digits and combining marks. + * Java identifiers (`Character.isJavaIdentifierStart`/`isJavaIdentifierPart`) permit a + * Unicode letter (Lu/Ll/Lt/Lm/Lo), a letter-number (Nl, e.g. Roman numerals), a currency + * symbol (Sc, e.g. `$`), or a connecting-punctuation character (Pc, e.g. `_`) at every + * position; continuation additionally allows decimal digits (Nd) and identifier-ignorable + * formatting characters (Cf, e.g. ZWNJ/ZWJ). Combining marks and non-decimal number + * categories (No) are not part of the Java grammar. */ -export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Sc}\p{Pc}$][\p{L}\p{N}\p{Sc}\p{Pc}\p{Mn}\p{Mc}$]*`; +export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Nl}\p{Sc}\p{Pc}][\p{L}\p{Nl}\p{Sc}\p{Pc}\p{Nd}\p{Cf}]*`; /** - * C# identifiers (ECMA-334 `identifier-start-character`/`identifier-part-character`) permit - * Unicode letter categories or an underscore at the first position, plus an optional leading - * `@` for a verbatim identifier (escaping a keyword, e.g. `@class`); continuation additionally - * allows decimal digits, connecting-punctuation, combining marks, and formatting characters. + * C# identifiers (ECMA-334 `identifier-start-character`/`identifier-part-character`) permit a + * Unicode letter (Lu/Ll/Lt/Lm/Lo), a letter-number (Nl), or a literal underscore at the first + * position, plus an optional leading `@` for a verbatim identifier (escaping a keyword, e.g. + * `@class`); continuation additionally allows decimal digits (Nd), connecting-punctuation + * (Pc), combining marks (Mn/Mc), and formatting characters (Cf). */ -export const CSHARP_IDENTIFIER_SOURCE = String.raw`@?[\p{L}\p{Pc}_][\p{L}\p{N}\p{Pc}\p{Mn}\p{Mc}\p{Cf}_]*`; +export const CSHARP_IDENTIFIER_SOURCE = String.raw`@?[\p{L}\p{Nl}_][\p{L}\p{Nl}_\p{Nd}\p{Pc}\p{Mn}\p{Mc}\p{Cf}]*`; diff --git a/tests/import-extraction-unicode-identifiers.test.ts b/tests/import-extraction-unicode-identifiers.test.ts index 6d736bef..0c9a8fbf 100644 --- a/tests/import-extraction-unicode-identifiers.test.ts +++ b/tests/import-extraction-unicode-identifiers.test.ts @@ -92,6 +92,18 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { imported: "$Widget", isStatic: false, }); + // Character.isJavaIdentifierStart accepts a letter-number (Nl) such as a Roman numeral, + // and isJavaIdentifierPart accepts an identifier-ignorable formatting character (Cf, + // e.g. ZWNJ) in continuation. + expect(parseJavaImportStatement("import com.example.\u2160Widget\u200c;")).toEqual({ + kind: "named", + from: "com.example.\u2160Widget\u200c", + imported: "\u2160Widget\u200c", + isStatic: false, + }); + // A non-decimal number character (No, e.g. the "½" fraction) is not accepted by + // isJavaIdentifierPart and must not be folded into the imported name. + expect(parseJavaImportStatement("import com.example.Widget\u00bd;")).toBeNull(); }); it("C#: using alias to a Unicode-named alias", () => { @@ -107,6 +119,16 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { alias: "@class", isStatic: false, }); + // ECMA-334 identifier-start-character accepts a letter-number (Nl, e.g. a Roman numeral) + // and identifier-part-character accepts a combining mark (Mn) in continuation. + expect(parseCsharpUsingDirective("using \u2160Alias = Some.cafe\u0301;")).toEqual({ + from: "Some.cafe\u0301", + alias: "\u2160Alias", + isStatic: false, + }); + // A connecting-punctuation character other than `_` (e.g. U+203F UNDERTIE) is a valid + // identifier-part-character but not a valid identifier-start-character. + expect(parseCsharpUsingDirective("using \u203fname = Some.Namespace;")).toBeNull(); }); it("Python fallback module-specifier extraction: import/from with Unicode module names", () => { expect(extractPythonSpecifiers("import créer\n")).toEqual(["créer"]); From e1fce21ee995b76442fc24ed2ed80876f842801a Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 18:27:30 -0400 Subject: [PATCH 24/29] fix: include combining marks in Java identifier continuation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Character.isJavaIdentifierPart accepts Mn/Mc combining marks (e.g. a decomposed 'café' written as 'cafe' + U+0301); JAVA_IDENTIFIER_SOURCE dropped them, so imports using decomposed Unicode names were silently truncated. Add a regression covering the decomposed form and document combining marks in the language-parity note. --- docs/language-parity.md | 2 +- src/util/identifiers.ts | 6 +++--- tests/import-extraction-unicode-identifiers.test.ts | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/language-parity.md b/docs/language-parity.md index f274b9a8..267816e0 100644 --- a/docs/language-parity.md +++ b/docs/language-parity.md @@ -92,7 +92,7 @@ Notes: - JavaScript expands `module.exports = { ...source }` for statically resolvable CommonJS imports and local object literals. Dynamic spread sources remain explicit namespace-reexport markers rather than silently disappearing. - Node `package.json#exports` condition matching follows author key order with mutually exclusive `import`/`require` modes threaded from ESM `import`/`import()` versus CommonJS `require()` (and TypeScript `import x = require(...)`). Nested conditions, array fallbacks, and `default` termination match Node for those cases. Custom `--conditions`, `browser`/`types`/`development`/`production`, import attributes, and `#imports` maps are not modeled. - Ruby treats `Constant = Struct.new(...)` as a class-kind symbol and a synthetic detailed class declaration. Runtime-computed class factories remain outside this recognition. -- Import and alias binding extraction accepts each source language's real identifier grammar rather than an ASCII-narrowed approximation: JS/TS/TSX use `ID_Start`/`ID_Continue` plus `$` and `_` (and ZWNJ/ZWJ continuations), Python uses PEP 3131 `XID_Start`/`XID_Continue`, Rust/Go/Kotlin use their Unicode letter/XID identifier rules, Java follows `Character.isJavaIdentifierStart`/`isJavaIdentifierPart` (Unicode letters, letter-numbers, currency symbols, connecting punctuation, decimal digits, and identifier-ignorable formatting characters), C# follows the ECMA-334 `identifier-start-character`/`identifier-part-character` grammar including the `@` verbatim-identifier prefix, and PHP accepts any byte `>= 0x80` at any identifier position. This covers native-query statement parsing, text/regex fallback recovery, and specifier extraction for those languages. +- Import and alias binding extraction accepts each source language's real identifier grammar rather than an ASCII-narrowed approximation: JS/TS/TSX use `ID_Start`/`ID_Continue` plus `$` and `_` (and ZWNJ/ZWJ continuations), Python uses PEP 3131 `XID_Start`/`XID_Continue`, Rust/Go/Kotlin use their Unicode letter/XID identifier rules, Java follows `Character.isJavaIdentifierStart`/`isJavaIdentifierPart` (Unicode letters, letter-numbers, currency symbols, connecting punctuation, decimal digits, combining marks, and identifier-ignorable formatting characters), C# follows the ECMA-334 `identifier-start-character`/`identifier-part-character` grammar including the `@` verbatim-identifier prefix, and PHP accepts any byte `>= 0x80` at any identifier position. This covers native-query statement parsing, text/regex fallback recovery, and specifier extraction for those languages. ## Project file discovery coverage diff --git a/src/util/identifiers.ts b/src/util/identifiers.ts index 34f5dc24..874b1e2a 100644 --- a/src/util/identifiers.ts +++ b/src/util/identifiers.ts @@ -18,11 +18,11 @@ export const PHP_IDENTIFIER_SOURCE = String.raw`[A-Za-z_\u{80}-\u{10FFFF}][A-Za- * Java identifiers (`Character.isJavaIdentifierStart`/`isJavaIdentifierPart`) permit a * Unicode letter (Lu/Ll/Lt/Lm/Lo), a letter-number (Nl, e.g. Roman numerals), a currency * symbol (Sc, e.g. `$`), or a connecting-punctuation character (Pc, e.g. `_`) at every - * position; continuation additionally allows decimal digits (Nd) and identifier-ignorable - * formatting characters (Cf, e.g. ZWNJ/ZWJ). Combining marks and non-decimal number + * position; continuation additionally allows decimal digits (Nd), combining marks (Mn/Mc), + * and identifier-ignorable formatting characters (Cf, e.g. ZWNJ/ZWJ). Non-decimal number * categories (No) are not part of the Java grammar. */ -export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Nl}\p{Sc}\p{Pc}][\p{L}\p{Nl}\p{Sc}\p{Pc}\p{Nd}\p{Cf}]*`; +export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Nl}\p{Sc}\p{Pc}][\p{L}\p{Nl}\p{Sc}\p{Pc}\p{Nd}\p{Mn}\p{Mc}\p{Cf}]*`; /** * C# identifiers (ECMA-334 `identifier-start-character`/`identifier-part-character`) permit a diff --git a/tests/import-extraction-unicode-identifiers.test.ts b/tests/import-extraction-unicode-identifiers.test.ts index 0c9a8fbf..56b9c0c1 100644 --- a/tests/import-extraction-unicode-identifiers.test.ts +++ b/tests/import-extraction-unicode-identifiers.test.ts @@ -104,6 +104,14 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { // A non-decimal number character (No, e.g. the "½" fraction) is not accepted by // isJavaIdentifierPart and must not be folded into the imported name. expect(parseJavaImportStatement("import com.example.Widget\u00bd;")).toBeNull(); + // isJavaIdentifierPart accepts combining marks (Mn/Mc); a decomposed identifier such as + // "café" written as "cafe" + combining acute accent (U+0301) is a single valid import. + expect(parseJavaImportStatement("import com.example.cafe\u0301;")).toEqual({ + kind: "named", + from: "com.example.cafe\u0301", + imported: "cafe\u0301", + isStatic: false, + }); }); it("C#: using alias to a Unicode-named alias", () => { From 5699b7e4923e023621a5027d2902866c8ebb22a7 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Sun, 16 Aug 2026 23:17:24 -0400 Subject: [PATCH 25/29] fix: disambiguate diff --git header paths using --- / +++ lines The unquoted-both-sides diff --git a/X b/Y header line is ambiguous when X or Y itself contains the literal text " b/" (Git never quotes plain spaces). finalizeFile now prefers the unambiguous single-path --- a/X and +++ b/Y lines (each carries exactly one path, so there is no split ambiguity) over the earliest-" b/"-split header guess, for added, deleted, modified, and rename/copy-fallback cases alike; the ambiguous split now only survives for pure renames/copies that have no content hunks and therefore no --- / +++ lines to correct it. Fixing this surfaced a second, previously-latent bug: real `git diff` appends a bare trailing tab to --- / +++ lines whenever the pathname contains a space (quoted or not), which broke decodeGitPath's closing-quote check once those lines started feeding path resolution directly. Strip that trailing tab before quote-decoding. --- docs/coverage/js.md | 6 +-- src/impact/parse.ts | 48 ++++++++++++++---- tests/streaming-parser.test.ts | 89 ++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 13 deletions(-) diff --git a/docs/coverage/js.md b/docs/coverage/js.md index 6e802d14..981e58e9 100644 --- a/docs/coverage/js.md +++ b/docs/coverage/js.md @@ -6,9 +6,9 @@ Source: `coverage/js/lcov.info` | Metric | Hit | Found | Coverage | | --------- | ----: | ----: | -------: | -| Lines | 27468 | 30241 | 90.83% | -| Functions | 4563 | 4842 | 94.24% | -| Branches | 20696 | 26100 | 79.30% | +| Lines | 27472 | 30245 | 90.83% | +| Functions | 4564 | 4843 | 94.24% | +| Branches | 20711 | 26117 | 79.30% | ## Least-covered Files diff --git a/src/impact/parse.ts b/src/impact/parse.ts index 9565538f..a3d45d0d 100644 --- a/src/impact/parse.ts +++ b/src/impact/parse.ts @@ -136,6 +136,15 @@ function stripDiffGitPrefix(pathValue: string, prefix: "a/" | "b/"): string { return pathValue.startsWith(prefix) ? pathValue.slice(prefix.length) : pathValue; } +// Git appends a bare trailing tab to `--- `/`+++ ` header lines whenever the pathname +// contains a space (quoted or not), to keep the path boundary unambiguous the way the +// traditional `diff -u` timestamp field did. It is a line-format marker, never part of the +// real filename, so strip it before quote-decoding: leaving it in place would make a quoted +// path fail `decodeGitPath`'s closing-quote check entirely. +function stripTrailingHeaderTab(rawPath: string): string { + return rawPath.endsWith("\t") ? rawPath.slice(0, -1) : rawPath; +} + function parseHeaderLine(currentFile: ParsedFileChange, line: string): void { if (line.startsWith("new file mode")) { currentFile._hasNewFileMode = true; @@ -179,11 +188,11 @@ function parseHeaderLine(currentFile: ParsedFileChange, line: string): void { return; } if (line.startsWith("--- ")) { - currentFile._fromPath = decodeGitPath(line.slice(4)); + currentFile._fromPath = decodeGitPath(stripTrailingHeaderTab(line.slice(4))); return; } if (line.startsWith("+++ ")) { - currentFile._toPath = decodeGitPath(line.slice(4)); + currentFile._toPath = decodeGitPath(stripTrailingHeaderTab(line.slice(4))); } } @@ -192,9 +201,12 @@ const QUOTED_PATH_SEGMENT = `"(?:[^"\\\\]|\\\\.)*"`; // Git quotes each side of the header independently, so a rename between an ASCII and a // non-ASCII path (or vice versa) can have only one side quoted. Quoted branches are tried // first since they are unambiguous (the closing quote is exact); the unquoted/unquoted -// fallback keeps the original earliest-" b/"-split heuristic, which is the best a regex can -// do when neither side is delimited (a literal " b/" inside an unquoted path is a -// pre-existing, accepted limitation, unchanged by this fix). +// fallback keeps the earliest-" b/"-split heuristic, which can pick the wrong boundary for +// an unquoted path that itself contains the literal text " b/". `buildInitiatedFile` stores +// this guess only as `_oldPathFromHeader`/`_newPathFromHeader`; `finalizeFile` overrides it +// with the unambiguous single-path `--- a/X`/`+++ b/Y` (and rename/copy from/to) lines +// whenever Git emits them, so the wrong split only survives for pure renames/copies that +// have no content hunks and therefore no `---`/`+++` lines to correct it. const DIFF_GIT_HEADER_BOTH_QUOTED = new RegExp(`^(${QUOTED_PATH_SEGMENT}) (${QUOTED_PATH_SEGMENT})$`); const DIFF_GIT_HEADER_A_QUOTED = new RegExp(`^(${QUOTED_PATH_SEGMENT}) b\\/(.+)$`); const DIFF_GIT_HEADER_B_QUOTED = new RegExp(`^a\\/(.+?) (${QUOTED_PATH_SEGMENT})$`); @@ -242,17 +254,31 @@ function initiateHunk(line: string): Hunk | null { } function finalizeFile(file: ParsedFileChange): void { - const renameFrom = file._renameFrom ?? file._oldPathFromHeader; - const renameTo = file._renameTo ?? file._newPathFromHeader; + // The `diff --git a/X b/Y` header line is ambiguous when both sides are unquoted and one + // side's path itself contains the literal separator text " b/" (e.g. a file named + // "foo b/bar"): the earliest-split fallback can pick the wrong boundary. The `--- a/X` and + // `+++ b/Y` lines each carry exactly one path with an unambiguous prefix, so prefer them + // (and the equally unambiguous rename/copy from/to lines) over the header split whenever + // Git emitted them; only fall back to the header split when no other source is available + // (pure renames/copies without content hunks omit `---`/`+++` entirely). + const unambiguousOldPath = + file._fromPath !== undefined && file._fromPath !== "/dev/null" + ? stripDiffGitPrefix(file._fromPath, "a/") + : undefined; + const unambiguousNewPath = + file._toPath !== undefined && file._toPath !== "/dev/null" ? stripDiffGitPrefix(file._toPath, "b/") : undefined; + + const renameFrom = file._renameFrom ?? unambiguousOldPath ?? file._oldPathFromHeader; + const renameTo = file._renameTo ?? unambiguousNewPath ?? file._newPathFromHeader; const copyFrom = file._copyFrom; - const copyTo = file._copyTo ?? file._newPathFromHeader; + const copyTo = file._copyTo ?? unambiguousNewPath ?? file._newPathFromHeader; if (file._hasNewFileMode || file._fromPath === "/dev/null") { file.kind = "added"; - file.path = file._newPathFromHeader ?? file.path; + file.path = unambiguousNewPath ?? file._newPathFromHeader ?? file.path; } else if (file._hasDeletedFileMode || file._toPath === "/dev/null") { file.kind = "deleted"; - file.path = file._oldPathFromHeader ?? file.path; + file.path = unambiguousOldPath ?? file._oldPathFromHeader ?? file.path; } else if (copyFrom && copyTo) { file.kind = "added"; file.path = copyTo; @@ -261,6 +287,8 @@ function finalizeFile(file: ParsedFileChange): void { file.kind = "renamed"; file.path = renameTo; file.oldPath = renameFrom; + } else { + file.path = unambiguousNewPath ?? file.path; } if (file._isBinary) { diff --git a/tests/streaming-parser.test.ts b/tests/streaming-parser.test.ts index 9626a6be..4837873e 100644 --- a/tests/streaming-parser.test.ts +++ b/tests/streaming-parser.test.ts @@ -229,4 +229,93 @@ copy to "copied\\t.ts" expect.objectContaining({ kind: "added", path: "copied\t.ts", oldPath: "source\t.ts" }), ]); }); + + it("disambiguates an unquoted path containing the literal header separator text using --- and +++", () => { + // Git does not C-quote a plain space, so a real filename containing " b/" makes the + // `diff --git a/X b/Y` line ambiguous at the regex level (multiple valid " b/" splits). + // The single-path `---`/`+++` lines are never ambiguous and must win over the header + // guess. + const diffText = [ + "diff --git a/foo b/bar b/foo b/bar", + "index 0000000..1111111 100644", + "--- a/foo b/bar", + "+++ b/foo b/bar", + "@@ -1 +1 @@", + "-old", + "+new", + "", + ].join("\n"); + + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: "foo b/bar", kind: "modified" })]); + }); + + it("disambiguates an ambiguous added-file path using the +++ header", () => { + const diffText = [ + "diff --git a/new b/file.ts b/new b/file.ts", + "new file mode 100644", + "index 0000000..1111111", + "--- /dev/null", + "+++ b/new b/file.ts", + "@@ -0,0 +1 @@", + "+export const value = 1;", + "", + ].join("\n"); + + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: "new b/file.ts", kind: "added" })]); + }); + + it("disambiguates an ambiguous deleted-file path using the --- header", () => { + const diffText = [ + "diff --git a/old b/file.ts b/old b/file.ts", + "deleted file mode 100644", + "index 1111111..0000000", + "--- a/old b/file.ts", + "+++ /dev/null", + "@@ -1 +0,0 @@", + "-export const value = 1;", + "", + ].join("\n"); + + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: "old b/file.ts", kind: "deleted" })]); + }); + + it("strips Git's trailing disambiguation tab from an unquoted --- / +++ path containing a space", () => { + // Real `git diff` appends a bare trailing tab to `---`/`+++` lines whenever the + // pathname contains a space, quoted or not (a holdover from the traditional `diff -u` + // timestamp field). It must not become part of the resolved path. + const diffText = [ + "diff --git a/with space.ts b/with space.ts", + "index 0000000..1111111 100644", + "--- a/with space.ts\t", + "+++ b/with space.ts\t", + "@@ -1 +1 @@", + "-old", + "+new", + "", + ].join("\n"); + + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: "with space.ts", kind: "modified" })]); + }); + + it("strips Git's trailing disambiguation tab from a quoted --- / +++ path", () => { + // The trailing tab sits outside the closing quote, so leaving it in place would make + // decodeGitPath's `endsWith('"')` check fail and return the raw quoted text unparsed. + const diffText = [ + 'diff --git "a/caf\\303\\251 with space.ts" "b/caf\\303\\251 with space.ts"', + "index 0000000..1111111 100644", + '--- "a/caf\\303\\251 with space.ts"\t', + '+++ "b/caf\\303\\251 with space.ts"\t', + "@@ -1 +1 @@", + "-old", + "+new", + "", + ].join("\n"); + + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: "café with space.ts", kind: "modified" })]); + }); }); From 9f5389adf103815f5d4d801cb34c49a2ad4c66c6 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 01:33:58 -0400 Subject: [PATCH 26/29] fix: address 4th review batch (Kotlin/Go identifiers, diff header disambiguation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Kotlin (native parseKotlinImportStatement + text fallback) now uses a per-segment KOTLIN_IDENTIFIER_SOURCE/KOTLIN_DOTTED_NAME_SOURCE (Kotlin's actual Letter/UnicodeDigit grammar: Lu/Ll/Lt/Lm/Lo + '_' at every position, Nd digits in continuation only) instead of a \p{N}-spanning class that accepted invalid statements like 'pkg.2mod' and 'Widget½'. - Go alias parsing (normalizeGoImports fallback and the separate parseGoImportAlias helper in indexer/shared.ts, both untouched by the earlier Unicode-breadth pass) now uses GO_IDENTIFIER_SOURCE (Go's real unicode_letter/unicode_digit grammar: L* + '_' start, Nd-only digit continuation) and treats the dot-import token '.' as a standalone alternative instead of an identifier prefix, so '.alias' and 'a½' are correctly rejected. - initiateFile's ambiguous-unquoted-header fallback now tries every ' b/' split position and prefers the one whose two halves are equal (the only case that reaches this fallback without --- / +++ or rename/copy markers to disambiguate), instead of always taking the earliest split; this also resolves the case where Git emits no --- / +++ lines at all (binary or mode-only changes). --- docs/coverage/js.md | 6 +-- src/impact/parse.ts | 44 +++++++++++++++---- src/indexer/imports/languageSpecific.ts | 17 +++++-- src/indexer/shared.ts | 6 ++- src/languages/importStatementParsers.ts | 11 +++-- src/util/identifiers.ts | 16 +++++++ tests/coverage-targeted.test.ts | 6 +++ ...ort-extraction-unicode-identifiers.test.ts | 6 +++ tests/streaming-parser.test.ts | 10 +++++ 9 files changed, 102 insertions(+), 20 deletions(-) diff --git a/docs/coverage/js.md b/docs/coverage/js.md index 981e58e9..83311ce1 100644 --- a/docs/coverage/js.md +++ b/docs/coverage/js.md @@ -6,9 +6,9 @@ Source: `coverage/js/lcov.info` | Metric | Hit | Found | Coverage | | --------- | ----: | ----: | -------: | -| Lines | 27472 | 30245 | 90.83% | -| Functions | 4564 | 4843 | 94.24% | -| Branches | 20711 | 26117 | 79.30% | +| Lines | 27489 | 30262 | 90.84% | +| Functions | 4565 | 4844 | 94.24% | +| Branches | 20716 | 26123 | 79.30% | ## Least-covered Files diff --git a/src/impact/parse.ts b/src/impact/parse.ts index a3d45d0d..35f599b6 100644 --- a/src/impact/parse.ts +++ b/src/impact/parse.ts @@ -201,16 +201,44 @@ const QUOTED_PATH_SEGMENT = `"(?:[^"\\\\]|\\\\.)*"`; // Git quotes each side of the header independently, so a rename between an ASCII and a // non-ASCII path (or vice versa) can have only one side quoted. Quoted branches are tried // first since they are unambiguous (the closing quote is exact); the unquoted/unquoted -// fallback keeps the earliest-" b/"-split heuristic, which can pick the wrong boundary for -// an unquoted path that itself contains the literal text " b/". `buildInitiatedFile` stores -// this guess only as `_oldPathFromHeader`/`_newPathFromHeader`; `finalizeFile` overrides it -// with the unambiguous single-path `--- a/X`/`+++ b/Y` (and rename/copy from/to) lines -// whenever Git emits them, so the wrong split only survives for pure renames/copies that +// fallback below (`resolveAmbiguousHeaderPaths`) prefers the split whose halves are equal, +// which resolves the common same-path case even when an unquoted path itself contains the +// literal text " b/". `buildInitiatedFile` stores its guess only as +// `_oldPathFromHeader`/`_newPathFromHeader`; `finalizeFile` still overrides it with the +// unambiguous single-path `--- a/X`/`+++ b/Y` (and rename/copy from/to) lines whenever Git +// emits them, so a genuinely undecidable split only survives for pure renames/copies that // have no content hunks and therefore no `---`/`+++` lines to correct it. const DIFF_GIT_HEADER_BOTH_QUOTED = new RegExp(`^(${QUOTED_PATH_SEGMENT}) (${QUOTED_PATH_SEGMENT})$`); const DIFF_GIT_HEADER_A_QUOTED = new RegExp(`^(${QUOTED_PATH_SEGMENT}) b\\/(.+)$`); const DIFF_GIT_HEADER_B_QUOTED = new RegExp(`^a\\/(.+?) (${QUOTED_PATH_SEGMENT})$`); -const DIFF_GIT_HEADER_PLAIN = /^a\/(.+?) b\/(.+)$/; + +/** + * The unquoted/unquoted fallback for `diff --git a/X b/Y`: try every position where the + * text " b/" occurs and prefer the split whose two halves are literally equal, since a + * changed file's old and new paths are the same string in every case that reaches this + * fallback (Git always emits `rename from`/`rename to` or `copy from`/`copy to` lines + * instead when the paths genuinely differ). Only when no split produces equal halves - an + * undecidable case with no other information available - fall back to the earliest split. + */ +function resolveAmbiguousHeaderPaths(remainder: string): { aSpec: string; bSpec: string } | null { + if (!remainder.startsWith("a/")) return null; + const afterA = remainder.slice(2); + const separator = " b/"; + const splitIndices: number[] = []; + for (let index = afterA.indexOf(separator); index !== -1; index = afterA.indexOf(separator, index + 1)) { + splitIndices.push(index); + } + if (!splitIndices.length) return null; + + let chosen = splitIndices[0]!; + for (const index of splitIndices) { + if (afterA.slice(0, index) === afterA.slice(index + separator.length)) { + chosen = index; + break; + } + } + return { aSpec: `a/${afterA.slice(0, chosen)}`, bSpec: `b/${afterA.slice(chosen + separator.length)}` }; +} function buildInitiatedFile(aSpec: string, bSpec: string): ParsedFileChange { const aPath = stripDiffGitPrefix(decodeGitPath(aSpec), "a/"); @@ -238,9 +266,9 @@ function initiateFile(line: string): ParsedFileChange | null { const bQuoted = remainder.match(DIFF_GIT_HEADER_B_QUOTED); if (bQuoted) return buildInitiatedFile(`a/${bQuoted[1]}`, bQuoted[2]!); - const plain = remainder.match(DIFF_GIT_HEADER_PLAIN); + const plain = resolveAmbiguousHeaderPaths(remainder); if (!plain) return null; - return buildInitiatedFile(`a/${plain[1]}`, `b/${plain[2]}`); + return buildInitiatedFile(plain.aSpec, plain.bSpec); } function initiateHunk(line: string): Hunk | null { diff --git a/src/indexer/imports/languageSpecific.ts b/src/indexer/imports/languageSpecific.ts index 675b52a8..bffd8e85 100644 --- a/src/indexer/imports/languageSpecific.ts +++ b/src/indexer/imports/languageSpecific.ts @@ -1,12 +1,14 @@ import path from "node:path"; import { JAVA_DOTTED_NAME_SOURCE, + KOTLIN_DOTTED_NAME_SOURCE, parseCsharpUsingDirective, parseJavaImportStatement, parseKotlinImportStatement, parsePhpImportStatement, parseRustImportStatement, } from "../../languages/importStatementParsers.js"; +import { GO_IDENTIFIER_SOURCE, KOTLIN_IDENTIFIER_SOURCE } from "../../util/identifiers.js"; import { isRustCfgTestStatement } from "../../util/rustTestModules.js"; import { getPhpComposerImplicitFiles } from "../../util/resolution.js"; import type { ImportBinding } from "../types.js"; @@ -38,8 +40,13 @@ function normalizeGoImports(context: LanguageSpecificImportContext): void { return; } const aliasByFrom = new Map(); - // Go identifiers permit Unicode letters (per the Go spec's "letter" production), not just ASCII. - const importPattern = /^\s*(?:import\s+)?(?:(?[._\p{L}][\p{L}\p{N}_]*)\s+)?["'`](?[^"'`]+)["'`]/gmu; + // Go alias is either the standalone dot-import token or a real Go identifier (Unicode + // letter/underscore start, decimal-digit continuation); the blank identifier "_" is a + // valid identifier already covered by GO_IDENTIFIER_SOURCE. + const importPattern = new RegExp( + String.raw`^\s*(?:import\s+)?(?:(?\.|${GO_IDENTIFIER_SOURCE})\s+)?["'\u0060](?[^"'\u0060]+)["'\u0060]`, + "gmu", + ); for (const match of context.source.matchAll(importPattern)) { const from = match.groups?.from; if (!from) continue; @@ -126,8 +133,10 @@ async function appendKotlinTextImports(context: LanguageSpecificImportContext): if (context.languageId !== "kotlin" || context.getBindings().length) { return; } - // Kotlin identifiers permit Unicode letters, not just ASCII. - const importPattern = /^\s*import\s+([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?\s*$/gmu; + const importPattern = new RegExp( + String.raw`^\s*import\s+(${KOTLIN_DOTTED_NAME_SOURCE}(?:\.\*)?)(?:\s+as\s+(${KOTLIN_IDENTIFIER_SOURCE}))?\s*$`, + "gmu", + ); for (const match of context.source.matchAll(importPattern)) { const rawSpec = match[1]; if (!rawSpec) continue; diff --git a/src/indexer/shared.ts b/src/indexer/shared.ts index 91c695d0..3699578e 100644 --- a/src/indexer/shared.ts +++ b/src/indexer/shared.ts @@ -1,10 +1,14 @@ +import { GO_IDENTIFIER_SOURCE } from "../util/identifiers.js"; + export { compareEdges, edgeKey, toRelativeEdge } from "../util/graphEdges.js"; export const DEFAULT_REF_CONTEXT_LINES = 5; +const GO_IMPORT_ALIAS_PATTERN = new RegExp(String.raw`^(\.|${GO_IDENTIFIER_SOURCE})\s+["'\u0060]`, "u"); + export function parseGoImportAlias(stmtText: string): string | null { const trimmed = stmtText.trim(); const importBody = trimmed.replace(/^import\s+/, ""); - const match = importBody.match(/^([._A-Za-z][\w]*)\s+["'`]/); + const match = importBody.match(GO_IMPORT_ALIAS_PATTERN); return match?.[1] ?? null; } diff --git a/src/languages/importStatementParsers.ts b/src/languages/importStatementParsers.ts index 1ea1057c..096f924f 100644 --- a/src/languages/importStatementParsers.ts +++ b/src/languages/importStatementParsers.ts @@ -2,6 +2,7 @@ import path from "node:path"; import { CSHARP_IDENTIFIER_SOURCE, JAVA_IDENTIFIER_SOURCE, + KOTLIN_IDENTIFIER_SOURCE, PHP_IDENTIFIER_SOURCE, XID_IDENTIFIER_SOURCE, } from "../util/identifiers.js"; @@ -365,6 +366,11 @@ function resolvePhpIncludePath(expr: string, fromFile?: string): string | null { return `./${relativePath}`; } +export const KOTLIN_DOTTED_NAME_SOURCE = String.raw`${KOTLIN_IDENTIFIER_SOURCE}(?:\.${KOTLIN_IDENTIFIER_SOURCE})*`; +const KOTLIN_IMPORT_PATTERN = new RegExp( + String.raw`^\s*import\s+(${KOTLIN_DOTTED_NAME_SOURCE}(?:\.\*)?)(?:\s+as\s+(${KOTLIN_IDENTIFIER_SOURCE}))?\s*$`, + "mu", +); export type ParsedKotlinImportStatement = | { kind: "named"; @@ -378,10 +384,7 @@ export type ParsedKotlinImportStatement = }; export function parseKotlinImportStatement(stmtText: string): ParsedKotlinImportStatement | null { - // Kotlin identifiers permit Unicode letters, not just ASCII. - const match = stmtText - .trim() - .match(/^\s*import\s+([\p{L}_][\p{L}\p{N}_.]*(?:\.\*)?)(?:\s+as\s+([\p{L}_][\p{L}\p{N}_]*))?\s*$/mu); + const match = stmtText.trim().match(KOTLIN_IMPORT_PATTERN); const rawSpec = match?.[1]; if (!rawSpec) return null; if (rawSpec.endsWith(".*")) { diff --git a/src/util/identifiers.ts b/src/util/identifiers.ts index 874b1e2a..6224ea5d 100644 --- a/src/util/identifiers.ts +++ b/src/util/identifiers.ts @@ -32,3 +32,19 @@ export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Nl}\p{Sc}\p{Pc}][\p{L} * (Pc), combining marks (Mn/Mc), and formatting characters (Cf). */ export const CSHARP_IDENTIFIER_SOURCE = String.raw`@?[\p{L}\p{Nl}_][\p{L}\p{Nl}_\p{Nd}\p{Pc}\p{Mn}\p{Mc}\p{Cf}]*`; + +/** + * Go identifiers (`unicode_letter`/`unicode_digit` in the Go spec's `identifier` production) + * permit a Unicode letter (Lu/Ll/Lt/Lm/Lo) or underscore at every position; continuation + * additionally allows decimal digits (Nd). Letter-numbers (Nl), other number categories + * (No), and combining marks are not part of the Go grammar. + */ +export const GO_IDENTIFIER_SOURCE = String.raw`[\p{L}_][\p{L}\p{Nd}_]*`; + +/** + * Kotlin identifiers (the `Letter`/`UnicodeDigit` lexer fragments in the Kotlin grammar) + * permit a Unicode letter (Lu/Ll/Lt/Lm/Lo) or underscore at every position; continuation + * additionally allows decimal digits (Nd). Letter-numbers (Nl), other number categories + * (No), and combining marks are not part of the Kotlin grammar. + */ +export const KOTLIN_IDENTIFIER_SOURCE = String.raw`[\p{L}_][\p{L}\p{Nd}_]*`; diff --git a/tests/coverage-targeted.test.ts b/tests/coverage-targeted.test.ts index 4f50117a..c5573b03 100644 --- a/tests/coverage-targeted.test.ts +++ b/tests/coverage-targeted.test.ts @@ -248,6 +248,12 @@ describe("targeted coverage for graph triples and native worker fallback", () => expect(parseGoImportAlias('import . "github.com/acme/pkg"')).toBe("."); expect(parseGoImportAlias('import _ "github.com/acme/pkg"')).toBe("_"); expect(parseGoImportAlias('import "fmt"')).toBeNull(); + // The dot-import token is standalone; ".alias" is not valid Go syntax and must not be + // captured as an identifier. + expect(parseGoImportAlias('import .alias "github.com/acme/pkg"')).toBeNull(); + // Go's unicode_digit is Nd only; a non-decimal number character (No, e.g. "½") is not a + // valid identifier continuation. + expect(parseGoImportAlias('import a\u00bd "github.com/acme/pkg"')).toBeNull(); expect(edgeKey(externalEdge)).toBe("C:/repo/src/main.ts|external:react|react|1"); expect(compareEdges(fileEdge, externalEdge)).toBeLessThan(0); expect(compareEdges(fileEdge, laterFileEdge)).toBeLessThan(0); diff --git a/tests/import-extraction-unicode-identifiers.test.ts b/tests/import-extraction-unicode-identifiers.test.ts index 56b9c0c1..b72b6532 100644 --- a/tests/import-extraction-unicode-identifiers.test.ts +++ b/tests/import-extraction-unicode-identifiers.test.ts @@ -75,6 +75,12 @@ describe("Import/alias extraction accepts Unicode identifiers", () => { imported: "Foo", local: "créer", }); + // A dotted segment must itself start with a valid identifier character; a digit + // immediately after "." is not part of Kotlin's grammar. + expect(parseKotlinImportStatement("import pkg.2mod")).toBeNull(); + // Kotlin's UnicodeDigit continuation is Nd only; a non-decimal number category (No, + // e.g. the "½" fraction) is not a valid identifier continuation. + expect(parseKotlinImportStatement("import com.example.Widget\u00bd")).toBeNull(); }); it("Java: import of a Unicode-named class", () => { diff --git a/tests/streaming-parser.test.ts b/tests/streaming-parser.test.ts index 4837873e..6b31c7d2 100644 --- a/tests/streaming-parser.test.ts +++ b/tests/streaming-parser.test.ts @@ -318,4 +318,14 @@ copy to "copied\\t.ts" const parsed = parseUnifiedDiff(diffText); expect(parsed.files).toEqual([expect.objectContaining({ path: "café with space.ts", kind: "modified" })]); }); + + it("resolves an ambiguous path from the diff --git header alone when Git emits no --- / +++ lines", () => { + // A mode-only change has no content hunks, so Git never emits --- / +++ lines to + // disambiguate; the equal-halves preference in the header split itself must still + // recover the real path "foo b/bar" instead of misreading it as a rename. + const diffText = ["diff --git a/foo b/bar b/foo b/bar", "old mode 100644", "new mode 100755", ""].join("\n"); + + const parsed = parseUnifiedDiff(diffText); + expect(parsed.files).toEqual([expect.objectContaining({ path: "foo b/bar", kind: "modified", modeChanged: true })]); + }); }); From 2f9f9b6eb87dcbe1cb6aedc73c94442d3d13a6a3 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 09:53:19 -0400 Subject: [PATCH 27/29] fix: add Java identifier-ignorable ISO control ranges; defer normalization and E2E fixtures - JAVA_IDENTIFIER_SOURCE now includes the ISO control ranges Character.isIdentifierIgnorable also accepts (U+0000-U+0008, U+000E-U+001B, U+007F-U+009F), matching the API named in its own doc comment exactly. Two remaining review comments (Rust/Python NFC/NFKC normalization for name resolution; end-to-end fixture coverage proving Unicode imports resolve through goto/references across all newly-broadened languages) are out of scope for this PR: both are net-new, multi-file work that goes beyond fixing the identifier-extraction regexes landed here, and a partial fix confined to this PR's existing diff would be incomplete or misleading. Documented as follow-up plans: - docs/plans/2026-08-17-unicode-identifier-normalization.md - docs/plans/2026-08-17-unicode-import-e2e-fixtures.md --- ...-08-17-unicode-identifier-normalization.md | 110 ++++++++++++++++++ .../2026-08-17-unicode-import-e2e-fixtures.md | 82 +++++++++++++ src/util/identifiers.ts | 7 +- 3 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 docs/plans/2026-08-17-unicode-identifier-normalization.md create mode 100644 docs/plans/2026-08-17-unicode-import-e2e-fixtures.md diff --git a/docs/plans/2026-08-17-unicode-identifier-normalization.md b/docs/plans/2026-08-17-unicode-identifier-normalization.md new file mode 100644 index 00000000..a8290702 --- /dev/null +++ b/docs/plans/2026-08-17-unicode-identifier-normalization.md @@ -0,0 +1,110 @@ +# Unicode identifier normalization for name resolution (2026-08-17) + +Status: Planned. Not started; no code in this plan has landed. + +## Problem + +PR #262 broadened import/alias extraction regexes (`src/util/identifiers.ts`, +`src/languages/importStatementParsers.ts`, `src/indexer/imports/*.ts`, +`src/graphs/specifiers.ts`, `src/util/specifiers.ts`) to accept each source +language's real identifier grammar, including combining-mark continuations +(Mn/Mc) for Java, C#, and PHP. + +Accepting a decomposed identifier at the regex layer is necessary but not +sufficient for correct resolution. Two languages have an explicit +normalization rule in their spec: + +- **Python (PEP 3131)**: identifiers are compared after NFKC normalization. + `café` (NFC, U+00E9) and `cafe\u0301` (NFD, "e" + combining acute) are the + _same_ identifier to CPython. +- **Rust**: `rustc` normalizes identifiers to NFC before name resolution + (tracked via `rustc_lexer`/`rustc_parse` identifier normalization since + Rust 1.0-era RFC on non-ASCII idents). Same NFC/NFD pair collapses to one + name. + +Java, C#, Kotlin, and Go do **not** normalize identifiers (each raw code +point sequence is a distinct identifier per their specs), so this plan is +scoped to Python and Rust only. + +Today, `codegraph` captures whatever byte sequence appears at each site +(import statement, declaration, reference) and compares those sequences +verbatim. A decomposed import (`import cafe\u0301`) will not resolve to a +composed declaration (`def café(): ...`) or vice versa, even though the +source language treats them as identical. This is a real, silent +navigation/reference gap, not a parsing gap — it cannot be fixed by +adjusting a regex character class. + +## Why this is a separate PR + +Fixing this only where PR #262 touched code (import binding extraction) +would be incomplete and misleading: it would make imports parse but not +resolve, or resolve inconsistently depending on which side of a match was +normalized. Correct behavior requires normalizing at every point a Python +or Rust identifier is captured or compared: + +1. **Import/alias extraction** (already regex-broadened in #262): + - `src/indexer/imports/python.ts` (`collectPythonImportsFromSource`) + - `src/graphs/specifiers.ts` (native Python `import`/`from` parsing) + - `src/util/specifiers.ts` (`extractPythonSpecifiers` fallback) + - `src/languages/importStatementParsers.ts` (`parseRustImportStatement`) +2. **Symbol declaration indexing** — not touched by #262, and the actual + source of the "declaration name" side of every match: + - `src/indexer/locals-and-exports.ts` (native capture → `SymbolDef.localName`) + - Wherever Rust/Python detailed symbol extraction reads a node's text as + a declaration name (`src/graphs/symbol-graph-detailed/*`, native query + capture text for `name`/`tname` captures). +3. **Navigation/resolution matching**: + - `src/indexer/navigation.ts` (`findReferences`) + - `src/indexer/navigation-resolve.ts` (`resolveExport`, import → declaration matching) + - `src/indexer/navigation-references.ts` (scope-based reference matching) + - `src/agent/renamePreview.ts`, `src/agent/refactorPlan.ts` (candidate + matching reuses the navigation layer, so should inherit this for free + once navigation normalizes) +4. **Symbol/reference hashing and IDs** — `defNodeId` in + `src/graphs/symbol-graph.ts` includes `localName` verbatim in the node + ID; normalizing only for comparison (not for the stored ID/display name) + avoids changing portable handles or displayed source text. + +## Proposed approach + +- Add `normalizeIdentifierForComparison(name: string, languageId: string): string` + to `src/util/identifiers.ts`. For `"python"` apply `name.normalize("NFKC")`; + for `"rust"` apply `name.normalize("NFC")`; for every other language return + `name` unchanged (explicit passthrough, not a default `.normalize()` call, + so adding a new language never silently opts in). +- Normalize **only at comparison sites**, never at storage sites: keep + `SymbolDef.localName`, import binding `imported`/`local`, and displayed + text exactly as they appear in source (required for accurate ranges, + rename edits, and portable handles). Build a normalized comparison key + alongside the raw name wherever lookups currently do `a === b` or + `map.get(name)` on a Python/Rust identifier, and use that key for the + lookup while keeping the raw name for everything else. +- Concretely: extend whatever lookup structure `resolveExport`/`findReferences` + use (name → declaration map) to key by `normalizeIdentifierForComparison` + instead of the raw string, for Python and Rust only. + +## Verification plan + +- Unit tests in `tests/import-extraction-unicode-identifiers.test.ts` + proving decomposed vs. composed import specs normalize to the same + extracted name (already partially covered for extraction; extend to + prove the _declaration_ side too). +- New cross-file fixture (see the companion E2E fixture plan + `2026-08-17-unicode-import-e2e-fixtures.md`) with a Python/Rust file + declaring a composed identifier and a consumer importing the decomposed + form (or vice versa), asserting `goto`/`references` resolve across the + pair. +- Explicit regression proving Java/C#/Kotlin/Go/PHP do **not** normalize + (a decomposed and composed Java identifier remain distinct symbols), + so this change cannot silently over-normalize those languages. +- Update `docs/language-parity.md`: state which languages normalize + identifiers for resolution (Python NFKC, Rust NFC) and which do not. + +## Non-goals + +- No change to displayed/stored identifier text, portable search handles, + or rename-edit content — normalization is comparison-only. +- No normalization for languages without a documented spec rule (Java, C#, + Kotlin, Go, PHP, JS/TS) even though their regex grammars now accept + combining marks; those combining-mark characters remain part of the + identifier's identity for those languages, matching their real compilers. diff --git a/docs/plans/2026-08-17-unicode-import-e2e-fixtures.md b/docs/plans/2026-08-17-unicode-import-e2e-fixtures.md new file mode 100644 index 00000000..b0538610 --- /dev/null +++ b/docs/plans/2026-08-17-unicode-import-e2e-fixtures.md @@ -0,0 +1,82 @@ +# End-to-end fixture coverage for Unicode import identifiers (2026-08-17) + +Status: Planned. Not started; no code in this plan has landed. + +## Problem + +PR #262 broadened import/alias extraction across JS/TS, Python, PHP, Rust, +Go, Java, Kotlin, and C# (`src/util/identifiers.ts` and the parsers/fallback +extractors that consume it) and added parser-level unit coverage in +`tests/import-extraction-unicode-identifiers.test.ts`. Those tests prove the +regexes and binding-construction functions accept/reject the right inputs +in isolation, matching each language's real identifier grammar. + +They do not prove a Unicode-named import survives the full pipeline: native +parse → import binding → graph edge → symbol declaration → `goto`/ +`references` resolution. Per `AGENTS.md`, "when adding or changing a +cross-file language scenario, add or update the nearest language test in +`tests/languages/*.test.ts` and the shared semantic coverage in +`tests/goto.test.ts`, `tests/references.test.ts`, and +`tests/native-semantic-parity.test.ts` when the language uses the native +runtime" — this PR's identifier-breadth change qualifies and that coverage +is currently missing. + +## Scope + +One cross-file scenario per already-native language touched by the +identifier-breadth work, following the existing fixture pattern in +`tests/samples//` (see `tests/samples/python/.regressions/ +unicode_def.py` / `unicode_consumer.py`, already added by this PR, as the +template): + +| Language | Sample directory | Unicode case to cover | +| -------- | ------------------------------------ | ------------------------------------------------------------------------------------------ | +| Java | `tests/samples/java/.regressions/` | `$`-prefixed and combining-mark class/import name | +| Kotlin | `tests/samples/kotlin/.regressions/` | Unicode `import ... as alias` | +| C# | `tests/samples/csharp/.regressions/` | `using alias = Namespace;` with a combining-mark alias, plus `@class`-style verbatim alias | +| Go | `tests/samples/go/.regressions/` | Unicode-letter import alias | +| PHP | `tests/samples/php/.regressions/` | non-`\p{L}` `use ... as alias` (e.g. emoji) | +| Rust | `tests/samples/rust/.regressions/` | `use ... as alias` with XID continuation beyond `\p{L}`/`\p{N}` | + +JS/TS/TSX and Python already have adjacent native-semantic-parity coverage +from this PR (`tests/samples/python/.regressions/unicode_*.py`, +`tests/native-semantic-parity.test.ts` Python fixtures); this plan extends +the same pattern to the remaining six languages. + +## Per-language work (repeat for each row above) + +1. Add a two-file fixture: a declaration file with a Unicode-named + exported symbol, and a consumer file that imports it using the + Unicode form the corresponding parser fix now accepts. +2. Extend `tests/languages/.test.ts` with a case asserting the + dependency graph includes the edge between consumer and declaration + file (mirrors existing `LanguageTestDefinition` fixtures in that file). +3. Extend `tests/goto.test.ts` with a case asserting go-to-definition from + the consumer's Unicode-named reference resolves to the declaration. +4. Extend `tests/references.test.ts` with a case asserting the declaration + appears in `findReferences` results from the consumer's usage site. +5. If the language uses the native runtime (all six do), extend + `tests/native-semantic-parity.test.ts` with the same fixture pair so + native-mode regression coverage catches drift. +6. Add a `docs/scenario-catalog.md` row per language (companion to the + parser-level rows already added by PR #262) pointing at the new + `tests/languages/*.test.ts` case as the "Sample". + +## Verification plan + +- `npx vitest run tests/languages/.test.ts tests/goto.test.ts +tests/references.test.ts tests/native-semantic-parity.test.ts` per + language as each is added. +- Full `npm run check` once all six languages are covered. +- Confirm each new case fails against the pre-PR-#262 regex (sanity check + that the fixture actually exercises the fixed code path, not an + already-passing ASCII-only case). + +## Non-goals + +- No new fixtures for languages whose identifier grammar was not changed + by PR #262 (Ruby, Swift, Zig, C, C++, SQL, etc.). +- No fixture coverage for the NFC/NFKC normalization work — that is + tracked separately in `2026-08-17-unicode-identifier-normalization.md` + and should reuse this plan's fixture pattern for Python/Rust once it + lands, rather than duplicating fixture setup here. diff --git a/src/util/identifiers.ts b/src/util/identifiers.ts index 6224ea5d..7d029fef 100644 --- a/src/util/identifiers.ts +++ b/src/util/identifiers.ts @@ -19,10 +19,11 @@ export const PHP_IDENTIFIER_SOURCE = String.raw`[A-Za-z_\u{80}-\u{10FFFF}][A-Za- * Unicode letter (Lu/Ll/Lt/Lm/Lo), a letter-number (Nl, e.g. Roman numerals), a currency * symbol (Sc, e.g. `$`), or a connecting-punctuation character (Pc, e.g. `_`) at every * position; continuation additionally allows decimal digits (Nd), combining marks (Mn/Mc), - * and identifier-ignorable formatting characters (Cf, e.g. ZWNJ/ZWJ). Non-decimal number - * categories (No) are not part of the Java grammar. + * and `Character.isIdentifierIgnorable` characters: formatting characters (Cf, e.g. + * ZWNJ/ZWJ) plus the ISO control ranges U+0000-U+0008, U+000E-U+001B, and U+007F-U+009F. + * Non-decimal number categories (No) are not part of the Java grammar. */ -export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Nl}\p{Sc}\p{Pc}][\p{L}\p{Nl}\p{Sc}\p{Pc}\p{Nd}\p{Mn}\p{Mc}\p{Cf}]*`; +export const JAVA_IDENTIFIER_SOURCE = String.raw`[\p{L}\p{Nl}\p{Sc}\p{Pc}][\p{L}\p{Nl}\p{Sc}\p{Pc}\p{Nd}\p{Mn}\p{Mc}\p{Cf}\u0000-\u0008\u000E-\u001B\u007F-\u009F]*`; /** * C# identifiers (ECMA-334 `identifier-start-character`/`identifier-part-character`) permit a From 34948847f872afeeb42f3002d80b0fe7575e0816 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 14:31:27 -0400 Subject: [PATCH 28/29] test: cover getGitBlobHashes for a newline-containing pathname; correct normalization plan scope - hashGitPaths exists specifically because 'hash-object --stdin-paths' newline-delimits input and cannot represent a pathname containing a newline; add a regression exercising that exact case (POSIX-only, NTFS cannot create such a filename). - The deferred identifier-canonicalization plan incorrectly excluded Java and C# by conflating 'no Unicode NFC/NFKC normalization' with 'no comparison-canonicalization rule'. Both languages have a real, spec-defined comparison rule distinct from full Unicode normalization (JLS Character.isIdentifierIgnorable stripping for Java; ECMA-334 '@' prefix + formatting-character stripping for C#) that the PR #262 identifier-breadth work now exposes a gap for. Corrected the plan's scope, file list, and verification plan to cover all four affected languages (Python, Rust, Java, C#) instead of two. --- ...-08-17-unicode-identifier-normalization.md | 139 +++++++++++------- tests/cache-invalidation.test.ts | 24 +++ 2 files changed, 109 insertions(+), 54 deletions(-) diff --git a/docs/plans/2026-08-17-unicode-identifier-normalization.md b/docs/plans/2026-08-17-unicode-identifier-normalization.md index a8290702..89d251a7 100644 --- a/docs/plans/2026-08-17-unicode-identifier-normalization.md +++ b/docs/plans/2026-08-17-unicode-identifier-normalization.md @@ -1,4 +1,4 @@ -# Unicode identifier normalization for name resolution (2026-08-17) +# Unicode identifier canonicalization for name resolution (2026-08-17) Status: Planned. Not started; no code in this plan has landed. @@ -8,103 +8,134 @@ PR #262 broadened import/alias extraction regexes (`src/util/identifiers.ts`, `src/languages/importStatementParsers.ts`, `src/indexer/imports/*.ts`, `src/graphs/specifiers.ts`, `src/util/specifiers.ts`) to accept each source language's real identifier grammar, including combining-mark continuations -(Mn/Mc) for Java, C#, and PHP. +(Mn/Mc) for Java, C#, and PHP, and identifier-ignorable formatting/control +characters (Cf, plus a handful of ISO control ranges) for Java. -Accepting a decomposed identifier at the regex layer is necessary but not -sufficient for correct resolution. Two languages have an explicit -normalization rule in their spec: +Accepting a wider identifier at the regex layer is necessary but not +sufficient for correct resolution: four languages define two spellings of +"the same" identifier as equal for name-resolution purposes, and +`codegraph` currently compares raw captured text everywhere, so it will +treat those equal spellings as different symbols. - **Python (PEP 3131)**: identifiers are compared after NFKC normalization. `café` (NFC, U+00E9) and `cafe\u0301` (NFD, "e" + combining acute) are the _same_ identifier to CPython. - **Rust**: `rustc` normalizes identifiers to NFC before name resolution (tracked via `rustc_lexer`/`rustc_parse` identifier normalization since - Rust 1.0-era RFC on non-ASCII idents). Same NFC/NFD pair collapses to one - name. - -Java, C#, Kotlin, and Go do **not** normalize identifiers (each raw code -point sequence is a distinct identifier per their specs), so this plan is -scoped to Python and Rust only. + the RFC on non-ASCII idents). Same NFC/NFD pair collapses to one name. +- **Java (JLS §3.8)**: two identifiers are the same "if, after ignoring + characters for which `Character.isIdentifierIgnorable` returns true, they + have the same sequence of characters." This is not Unicode normalization; + it is deletion of `Cf` formatting characters (ZWNJ/ZWJ/bidi/etc.) and a + handful of ISO control ranges — the exact character set + `JAVA_IDENTIFIER_SOURCE` (added in #262) now accepts as legal continuation + characters. `Foo` and `Foo\u200C` are the same field to `javac` but would + currently resolve as two different symbols here. +- **C# (ECMA-334)**: two identifiers match if identical after (1) removing + a leading `@` verbatim-identifier prefix, (2) resolving + unicode-escape-sequences, and (3) removing `Cf` formatting characters. + `@Widget` and `Widget` name the same symbol; `Widget` and `Widget\u200C` + do too. `CSHARP_IDENTIFIER_SOURCE` (added in #262) accepts both the `@` + prefix and `Cf` continuation but nothing downstream removes them for + comparison. + +Kotlin and Go have no such rule (raw code point sequences are compared +directly per their specs, and neither grammar admits `Cf`/ignorable +characters at all), so this plan does not touch them. PHP compares raw +bytes with no normalization step either. JS/TS (ECMAScript) also performs +no identifier normalization for name resolution — two different Unicode +spellings are genuinely different bindings. Today, `codegraph` captures whatever byte sequence appears at each site (import statement, declaration, reference) and compares those sequences -verbatim. A decomposed import (`import cafe\u0301`) will not resolve to a -composed declaration (`def café(): ...`) or vice versa, even though the -source language treats them as identical. This is a real, silent -navigation/reference gap, not a parsing gap — it cannot be fixed by -adjusting a regex character class. +verbatim. This is a real, silent navigation/reference gap, not a parsing +gap — it cannot be fixed by adjusting a regex character class. ## Why this is a separate PR Fixing this only where PR #262 touched code (import binding extraction) would be incomplete and misleading: it would make imports parse but not resolve, or resolve inconsistently depending on which side of a match was -normalized. Correct behavior requires normalizing at every point a Python -or Rust identifier is captured or compared: +canonicalized. Correct behavior requires canonicalizing at every point a +Python, Rust, Java, or C# identifier is captured or compared: 1. **Import/alias extraction** (already regex-broadened in #262): - `src/indexer/imports/python.ts` (`collectPythonImportsFromSource`) - `src/graphs/specifiers.ts` (native Python `import`/`from` parsing) - `src/util/specifiers.ts` (`extractPythonSpecifiers` fallback) - - `src/languages/importStatementParsers.ts` (`parseRustImportStatement`) + - `src/languages/importStatementParsers.ts` (`parseRustImportStatement`, + `parseJavaImportStatement`, `parseCsharpUsingDirective`) + - `src/indexer/imports/languageSpecific.ts` (Java text fallback) 2. **Symbol declaration indexing** — not touched by #262, and the actual source of the "declaration name" side of every match: - `src/indexer/locals-and-exports.ts` (native capture → `SymbolDef.localName`) - - Wherever Rust/Python detailed symbol extraction reads a node's text as - a declaration name (`src/graphs/symbol-graph-detailed/*`, native query - capture text for `name`/`tname` captures). + - Wherever Rust/Python/Java/C# detailed symbol extraction reads a node's + text as a declaration name (`src/graphs/symbol-graph-detailed/*`, + native query capture text for `name`/`tname` captures). 3. **Navigation/resolution matching**: - `src/indexer/navigation.ts` (`findReferences`) - `src/indexer/navigation-resolve.ts` (`resolveExport`, import → declaration matching) - `src/indexer/navigation-references.ts` (scope-based reference matching) - `src/agent/renamePreview.ts`, `src/agent/refactorPlan.ts` (candidate matching reuses the navigation layer, so should inherit this for free - once navigation normalizes) + once navigation canonicalizes) 4. **Symbol/reference hashing and IDs** — `defNodeId` in `src/graphs/symbol-graph.ts` includes `localName` verbatim in the node - ID; normalizing only for comparison (not for the stored ID/display name) - avoids changing portable handles or displayed source text. + ID; canonicalizing only for comparison (not for the stored ID/display + name) avoids changing portable handles or displayed source text. ## Proposed approach -- Add `normalizeIdentifierForComparison(name: string, languageId: string): string` - to `src/util/identifiers.ts`. For `"python"` apply `name.normalize("NFKC")`; - for `"rust"` apply `name.normalize("NFC")`; for every other language return - `name` unchanged (explicit passthrough, not a default `.normalize()` call, - so adding a new language never silently opts in). -- Normalize **only at comparison sites**, never at storage sites: keep +- Add `canonicalizeIdentifierForComparison(name: string, languageId: string): string` + to `src/util/identifiers.ts` with one explicit branch per language that + needs it, and an explicit passthrough default for every other language + (never a default `.normalize()`/strip call, so adding a new language + never silently opts in): + - `"python"`: `name.normalize("NFKC")`. + - `"rust"`: `name.normalize("NFC")`. + - `"java"`: strip every code point in the `JAVA_IDENTIFIER_SOURCE` + continuation class's `Cf`/ISO-control set (reuse the same ranges + documented on `JAVA_IDENTIFIER_SOURCE` so the two never drift apart). + - `"csharp"`/`"cs"`: strip a single leading `@`, then strip `Cf` + characters (reuse the `Cf` portion of `CSHARP_IDENTIFIER_SOURCE`). + - everything else: return `name` unchanged. +- Canonicalize **only at comparison sites**, never at storage sites: keep `SymbolDef.localName`, import binding `imported`/`local`, and displayed text exactly as they appear in source (required for accurate ranges, - rename edits, and portable handles). Build a normalized comparison key + rename edits, and portable handles). Build a canonicalized comparison key alongside the raw name wherever lookups currently do `a === b` or - `map.get(name)` on a Python/Rust identifier, and use that key for the - lookup while keeping the raw name for everything else. + `map.get(name)` on a Python/Rust/Java/C# identifier, and use that key for + the lookup while keeping the raw name for everything else. - Concretely: extend whatever lookup structure `resolveExport`/`findReferences` - use (name → declaration map) to key by `normalizeIdentifierForComparison` - instead of the raw string, for Python and Rust only. + use (name → declaration map) to key by + `canonicalizeIdentifierForComparison` instead of the raw string, for + Python, Rust, Java, and C# only. ## Verification plan - Unit tests in `tests/import-extraction-unicode-identifiers.test.ts` - proving decomposed vs. composed import specs normalize to the same - extracted name (already partially covered for extraction; extend to - prove the _declaration_ side too). -- New cross-file fixture (see the companion E2E fixture plan - `2026-08-17-unicode-import-e2e-fixtures.md`) with a Python/Rust file - declaring a composed identifier and a consumer importing the decomposed - form (or vice versa), asserting `goto`/`references` resolve across the - pair. -- Explicit regression proving Java/C#/Kotlin/Go/PHP do **not** normalize - (a decomposed and composed Java identifier remain distinct symbols), - so this change cannot silently over-normalize those languages. -- Update `docs/language-parity.md`: state which languages normalize - identifiers for resolution (Python NFKC, Rust NFC) and which do not. + proving each canonicalization branch collapses the documented equal + pairs (`café`/`cafe\u0301` for Python, `Foo`/`Foo\u200C` for Java, + `@Widget`/`Widget` for C#, NFC/NFD pairs for Rust) to the same key, + covering both the extraction and declaration side. +- New cross-file fixtures (see the companion E2E fixture plan + `2026-08-17-unicode-import-e2e-fixtures.md`) per canonicalizing language: + a declaration file using one spelling and a consumer importing the + equal-but-differently-spelled form, asserting `goto`/`references` + resolve across the pair. +- Explicit regression proving Kotlin/Go/PHP/JS/TS do **not** canonicalize + (two differently-spelled-but-"equal" forms remain distinct symbols for + those languages), so this change cannot silently over-canonicalize them. +- Update `docs/language-parity.md`: state which languages canonicalize + identifiers for resolution (Python NFKC, Rust NFC, Java + identifier-ignorable stripping, C# `@`-prefix + formatting-character + stripping) and which do not. ## Non-goals - No change to displayed/stored identifier text, portable search handles, - or rename-edit content — normalization is comparison-only. -- No normalization for languages without a documented spec rule (Java, C#, - Kotlin, Go, PHP, JS/TS) even though their regex grammars now accept - combining marks; those combining-mark characters remain part of the - identifier's identity for those languages, matching their real compilers. + or rename-edit content — canonicalization is comparison-only. +- No canonicalization for languages without a documented spec rule + (Kotlin, Go, PHP, JS/TS) even though PR #262 broadened their extraction + grammars; those characters remain part of the identifier's identity for + those languages, matching their real compilers. diff --git a/tests/cache-invalidation.test.ts b/tests/cache-invalidation.test.ts index fc655ec2..3f58d8cc 100644 --- a/tests/cache-invalidation.test.ts +++ b/tests/cache-invalidation.test.ts @@ -912,6 +912,30 @@ describe("Cache invalidation and strict hashing", () => { } }); + // `hash-object --stdin-paths` newline-delimits its input, so it cannot represent a + // pathname that itself contains a newline; this is the specific case the argv-based + // implementation in `hashGitPaths` exists to support. NTFS rejects `\n` in filenames, so + // this only runs on POSIX filesystems. + it.skipIf(process.platform === "win32")( + "returns a git signature for a tracked path containing a newline", + async () => { + const root = await mkTmpDir("dg-git-sig-newline-path-"); + runGit(root, ["init"]); + runGit(root, ["config", "user.email", "cache@test.local"]); + runGit(root, ["config", "user.name", "Cache Test"]); + + const filePath = path.join(root, "line1\nline2.ts"); + await fsp.writeFile(filePath, "export const value = 1;\n", "utf8"); + runGit(root, ["add", "-A"]); + runGit(root, ["commit", "-m", "newline path"]); + + const hashes = await gitModule.getGitBlobHashes(root, [filePath]); + + expect(hashes.size).toBe(1); + expect(hashes.get(normalize(filePath))).toMatch(/^[0-9a-f]{40}$/); + }, + ); + it("surfaces a genuine git invocation failure instead of silently discarding signatures", async () => { const root = await mkTmpDir("dg-git-sig-invocation-failure-"); // No `git init`: the directory is not a repository, so `git ls-files` genuinely fails From e4c37fd0249d82a6f9fc2e55bd18860a328414b2 Mon Sep 17 00:00:00 2001 From: Luke Zehrung Date: Mon, 17 Aug 2026 16:34:41 -0400 Subject: [PATCH 29/29] fix: convert AST-grep capture columns from UTF-8 bytes to UTF-16 streamAstGrep returned capture.start.column directly from the native query result, bypassing the byte-index-map conversion the rest of this PR applies everywhere else. A capture on a line with multibyte text before it reported a byte-based column, off by one UTF-16 code unit per extra byte the preceding multibyte text used, while textGrep and every other JS-facing API already report UTF-16 columns. Route it through stringPositionForBytePoint, built once per file, and add a regression proving the column matches source.indexOf() UTF-16 semantics. Also: drop an === true boolean comparison in tests/fast-graph-edgecases.test.ts per repo style. --- docs/coverage/js.md | 2 +- src/graphs/grep.ts | 9 +++++++-- tests/fast-graph-edgecases.test.ts | 2 +- tests/grep-default-patterns.test.ts | 22 ++++++++++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/coverage/js.md b/docs/coverage/js.md index 83311ce1..fcdfef37 100644 --- a/docs/coverage/js.md +++ b/docs/coverage/js.md @@ -6,7 +6,7 @@ Source: `coverage/js/lcov.info` | Metric | Hit | Found | Coverage | | --------- | ----: | ----: | -------: | -| Lines | 27489 | 30262 | 90.84% | +| Lines | 27491 | 30264 | 90.84% | | Functions | 4565 | 4844 | 94.24% | | Branches | 20716 | 26123 | 79.30% | diff --git a/src/graphs/grep.ts b/src/graphs/grep.ts index b4f5a213..514417b7 100644 --- a/src/graphs/grep.ts +++ b/src/graphs/grep.ts @@ -1,6 +1,7 @@ import fsp from "node:fs/promises"; import { prepareSourceInput } from "../languages/filePrep.js"; import { logWithLevel } from "../logging.js"; +import { buildByteToStringIndexMap, stringPositionForBytePoint } from "../native/byteIndex.js"; import { getUnifiedQueryExecution } from "../native/treeSitterNative.js"; import { toProjectDisplayPath } from "../util/paths.js"; import { listProjectFiles, type ProjectFileDiscoveryOptions } from "../util/projectFiles.js"; @@ -38,13 +39,17 @@ export async function* streamAstGrep( const matches = getUnifiedQueryExecution(source, support, querySource).matches; if (!matches) continue; + // Native captures expose UTF-8 byte offsets/columns; convert once per file so every + // capture reports the same UTF-16 column the rest of the JS APIs (and text grep) use. + const byteIndexMap = buildByteToStringIndexMap(source); for (const match of matches) { for (const capture of match.captures) { + const start = stringPositionForBytePoint(byteIndexMap, capture.start); yield { file: toProjectDisplayPath(projectRoot, file), capture: capture.name, - line: capture.start.row + 1, - column: capture.start.column + 1, + line: start.row + 1, + column: start.column + 1, snippet: capture.text.replace(/\n/g, " "), }; } diff --git a/tests/fast-graph-edgecases.test.ts b/tests/fast-graph-edgecases.test.ts index a4c9dd16..555023a1 100644 --- a/tests/fast-graph-edgecases.test.ts +++ b/tests/fast-graph-edgecases.test.ts @@ -44,7 +44,7 @@ describe("Fast graph edge cases", () => { // A separate runtime import (`{ f }`) and type-only import (`type { T }`) to the same // target module must both survive dedup, not collapse onto one entry. expect(toUtil).toHaveLength(2); - expect(toUtil.some((edge) => edge.typeOnly === true)).toBe(true); + expect(toUtil.some((edge) => edge.typeOnly)).toBe(true); expect(toUtil.some((edge) => !edge.typeOnly)).toBe(true); }); diff --git a/tests/grep-default-patterns.test.ts b/tests/grep-default-patterns.test.ts index 8ce6a434..cb3ba98f 100644 --- a/tests/grep-default-patterns.test.ts +++ b/tests/grep-default-patterns.test.ts @@ -27,4 +27,26 @@ describe("grep default patterns", () => { await fsp.rm(root, { recursive: true, force: true }); } }); + + it("reports UTF-16 columns for AST-grep captures after multibyte text on the same line (C11)", async () => { + const root = await mkTmpDir("cg-grep-multibyte-column-"); + const file = path.join(root, "entry.ts"); + + // "café" precedes the captured import source on the same line: "é" is one UTF-16 code + // unit but two UTF-8 bytes, so a byte-based column would report one column too far right. + const source = "const café = 1; import { helper } from './dep';\n"; + await fsp.writeFile(file, source, "utf8"); + const target = "'./dep'"; + const expectedColumn = source.indexOf(target) + 1; + + try { + const hits = await astGrep(root, "(import_statement source: (string) @mod)", ["**/*.ts"]); + + expect(hits).toEqual([ + expect.objectContaining({ capture: "mod", line: 1, column: expectedColumn, snippet: target }), + ]); + } finally { + await fsp.rm(root, { recursive: true, force: true }); + } + }); });