Switch tree-sitter to WASM runtime for cross-platform CI compatibility

Native tree-sitter required C++ compilation via node-gyp which failed on
GitHub Actions with Node 24 (V8 C++20 requirement). Switch to web-tree-sitter
(WASM) with grammar files from @vscode/tree-sitter-wasm.

- Replace native tree-sitter + grammar packages with web-tree-sitter + @vscode/tree-sitter-wasm
- Make parseSource/parseCodeFile/parseFile/extractSymbols async (WASM loading is async)
- Remove native tree-sitter jest mock, use --experimental-vm-modules instead
- Remove npm overrides (no more native peer dep conflicts)
- Update all tests for async parser API (beforeAll pattern)

Author: prih

package-lock.json

@@ -14,8 +14,8 @@
     "dev": "tsc --watch",
     "cli": "node dist/cli/index.js",
     "cli:dev": "tsx src/cli/index.ts",
-    "test": "jest",
-    "test:watch": "jest --watch"
+    "test": "NODE_OPTIONS='--experimental-vm-modules' jest",
+    "test:watch": "NODE_OPTIONS='--experimental-vm-modules' jest --watch"
   },
   "repository": {
     "type": "git",
@@ -49,6 +49,7 @@
   "dependencies": {
     "@huggingface/transformers": "^3.8.1",
     "@modelcontextprotocol/sdk": "^1.27.1",
+    "@vscode/tree-sitter-wasm": "^0.3.0",
     "chokidar": "^5.0.0",
     "commander": "^14.0.3",
     "cookie-parser": "^1.4.7",
@@ -60,9 +61,7 @@
     "micromatch": "^4.0.8",
     "mime": "^4.1.0",
     "multer": "^2.1.1",
-    "tree-sitter": "^0.25.0",
-    "tree-sitter-javascript": "^0.25.0",
-    "tree-sitter-typescript": "^0.23.2",
+    "web-tree-sitter": "^0.26.7",
     "ws": "^8.19.0",
     "yaml": "^2.8.2",
     "zod": "^4.3.6"
@@ -85,10 +84,5 @@
     "tsc-alias": "^1.8.16",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3"
-  },
-  "overrides": {
-    "tree-sitter-typescript": {
-      "tree-sitter": "$tree-sitter"
-    }
   }
 }

package.json

@@ -102,7 +102,7 @@ export function createProjectIndexer(
     const fileId = path.relative(config.projectDir, absolutePath);
     if (getFileMtime(docGraph, fileId) === mtime) return;
     const content = fs.readFileSync(absolutePath, 'utf-8');
-    const chunks = parseFile(content, absolutePath, config.projectDir, config.chunkDepth);
+    const chunks = await parseFile(content, absolutePath, config.projectDir, config.chunkDepth);
     // Batch-embed all chunks + file-level in one forward pass
     const batchInputs = chunks.map(c => ({ title: c.title, content: c.content }));
     const rootChunk = chunks.find(c => c.level === 1);
@@ -139,7 +139,7 @@ export function createProjectIndexer(
     const mtime = stat.mtimeMs;
     const fileId = path.relative(config.projectDir, absolutePath);
     if (getCodeFileMtime(codeGraph, fileId) === mtime) return;
-    const parsed = parseCodeFile(absolutePath, config.projectDir, mtime);
+    const parsed = await parseCodeFile(absolutePath, config.projectDir, mtime);
     // Batch-embed all symbols + file-level in one forward pass
     const batchInputs = parsed.nodes.map(({ attrs }) => ({ title: attrs.signature, content: attrs.docComment }));
     batchInputs.push({ title: fileId, content: '' });

src/cli/indexer.ts

@@ -52,11 +52,11 @@ function hasFile(p: string): boolean {
 // Main parser
 // ---------------------------------------------------------------------------
 
-export function parseCodeFile(
+export async function parseCodeFile(
   absolutePath: string,
   codeDir: string,
   mtime: number,
-): ParsedFile {
+): Promise<ParsedFile> {
   const fileId = path.relative(codeDir, absolutePath);
 
   // Determine language from file extension
@@ -77,7 +77,7 @@ export function parseCodeFile(
   }
 
   const source = fs.readFileSync(absolutePath, 'utf-8');
-  const rootNode = parseSource(source, language);
+  const rootNode = await parseSource(source, language);
 
   if (!rootNode) {
     return {

src/lib/parsers/code.ts

@@ -14,12 +14,12 @@ const TAG_TO_LANGUAGE: Record<string, string> = {
  * Extract top-level symbol names from a code block using tree-sitter.
  * Returns [] for unsupported languages or on parse failure.
  */
-export function extractSymbols(code: string, language: string): string[] {
+export async function extractSymbols(code: string, language: string): Promise<string[]> {
   const lang = TAG_TO_LANGUAGE[language.toLowerCase()];
   if (!lang || !isLanguageSupported(lang)) return [];
 
   try {
-    const rootNode = parseSource(code, lang);
+    const rootNode = await parseSource(code, lang);
     if (!rootNode) return [];
 
     const mapper = getMapper(lang)!;

src/lib/parsers/codeblock.ts

@@ -15,12 +15,12 @@ export interface Chunk {
 }
 
 // Parse a markdown file into chunks split by headings
-export function parseFile(
+export async function parseFile(
   content: string,
   absolutePath: string,
   projectDir: string,
   chunkDepth: number,
-): Chunk[] {
+): Promise<Chunk[]> {
   const fileId = path.relative(projectDir, absolutePath);
   const lines = content.split('\n');
 
@@ -65,14 +65,14 @@ export function parseFile(
       };
     });
 
-  return spliceCodeBlocks(textChunks, seenIds);
+  return await spliceCodeBlocks(textChunks, seenIds);
 }
 
 // --- code block extraction ---
 
 const FENCE_RE = /^(`{3,}|~{3,})(\S*)\s*\n([\s\S]*?)^\1\s*$/gm;
 
-function spliceCodeBlocks(chunks: Chunk[], seenIds: Set<string>): Chunk[] {
+async function spliceCodeBlocks(chunks: Chunk[], seenIds: Set<string>): Promise<Chunk[]> {
   const result: Chunk[] = [];
 
   for (const chunk of chunks) {
@@ -102,7 +102,7 @@ function spliceCodeBlocks(chunks: Chunk[], seenIds: Set<string>): Chunk[] {
       seenIds.add(id);
 
       const lang = cb.language || undefined;
-      const symbols = lang ? extractSymbols(cb.code, lang) : [];
+      const symbols = lang ? await extractSymbols(cb.code, lang) : [];
 
       result.push({
         id,

src/lib/parsers/docs.ts

@@ -1,4 +1,4 @@
-export { registerLanguage, getLanguageEntry, isLanguageSupported, parseSource, getMapper, listLanguages } from './registry';
+export { registerLanguage, isLanguageSupported, parseSource, getMapper, listLanguages, initParser } from './registry';
 export type { LanguageMapper, ExtractedSymbol, ExtractedEdge, ExtractedImport } from './types';
 export { registerTypescript } from './typescript';
 

src/lib/parsers/languages/index.ts

@@ -1,50 +1,77 @@
-import Parser from 'tree-sitter';
+import path from 'path';
 import type { LanguageMapper } from './types';
 
 export { type LanguageMapper, type ExtractedSymbol, type ExtractedEdge, type ExtractedImport } from './types';
 
+// web-tree-sitter types (loaded lazily)
+type WTSLanguage = any;
+
 interface LanguageEntry {
-  /** tree-sitter Language object */
-  grammar: any;
+  /** WASM file name, e.g. 'tree-sitter-typescript.wasm' */
+  wasmFile: string;
+  /** Loaded Language instance (null until loadLanguage is called) */
+  language: WTSLanguage | null;
   /** Mapper that extracts symbols, edges, imports from the AST */
   mapper: LanguageMapper;
 }
 
 /** Map from language name (matching file-lang.ts names) to entry. */
 const languages = new Map<string, LanguageEntry>();
 
-/** Shared parser instance — setLanguage() is cheap, no need for one per language. */
-let _parser: Parser | undefined;
+/** WASM directory containing grammar .wasm files */
+const WASM_DIR = path.join(
+  path.dirname(require.resolve('@vscode/tree-sitter-wasm/package.json')),
+  'wasm',
+);
+
+let _initPromise: Promise<void> | null = null;
+
+/** web-tree-sitter module (lazy loaded) */
+let _wts: any = null;
+
+/** Initialize the WASM parser runtime. Must be called before parsing. */
+export async function initParser(): Promise<void> {
+  if (_wts) return;
+  if (_initPromise) return _initPromise;
+
+  _initPromise = (async () => {
+    _wts = require('web-tree-sitter');
+    await _wts.Parser.init();
+  })();
 
-function getParser(): Parser {
-  if (!_parser) _parser = new Parser();
-  return _parser;
+  return _initPromise;
 }
 
-/** Register a language. Called at module load time by each language module. */
-export function registerLanguage(name: string, grammar: any, mapper: LanguageMapper): void {
-  languages.set(name, { grammar, mapper });
+/** Register a language (sync — only stores metadata). */
+export function registerLanguage(name: string, wasmFile: string, mapper: LanguageMapper): void {
+  languages.set(name, { wasmFile, language: null, mapper });
 }
 
-/** Get a registered language entry. Returns undefined for unsupported languages. */
-export function getLanguageEntry(languageName: string): LanguageEntry | undefined {
-  return languages.get(languageName);
+/** Load a language WASM if not already loaded. */
+async function loadLanguage(entry: LanguageEntry): Promise<WTSLanguage> {
+  if (entry.language) return entry.language;
+  await initParser();
+  const wasmPath = path.join(WASM_DIR, entry.wasmFile);
+  entry.language = await _wts.Language.load(wasmPath);
+  return entry.language;
 }
 
-/** Check if a language is supported. */
+/** Check if a language is registered. */
 export function isLanguageSupported(languageName: string): boolean {
   return languages.has(languageName);
 }
 
 /** Parse source code with the appropriate language grammar. Returns root node or null. */
-export function parseSource(code: string, languageName: string): any | null {
+export async function parseSource(code: string, languageName: string): Promise<any | null> {
   const entry = languages.get(languageName);
   if (!entry) return null;
 
-  const parser = getParser();
-  parser.setLanguage(entry.grammar);
+  await initParser();
+  const lang = await loadLanguage(entry);
+  const parser = new _wts.Parser();
+  parser.setLanguage(lang);
   const tree = parser.parse(code);
-  return tree.rootNode;
+  return tree?.rootNode ?? null;
 }
 
 /** Get the mapper for a language. Returns undefined for unsupported languages. */