fix(parsers): replace silent skip with regex fallback for large files (closes #163)

CHANGELOG.md

@@ -7,6 +7,52 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [8.2.0] — Unreleased
 
+### Large-File Silent-Skip Replaced with Explicit Regex Fallback (issue #163)
+
+CodeLens silently skipped files above hardcoded line thresholds to
+avoid a tree-sitter 0.26 binding segfault (tracked in #116):
+
+- JavaScript: files > 100 lines → skipped (returned `{"nodes": [], "edges": []}`)
+- Python: files > 200 lines → skipped
+
+This caused the most complex files in a codebase — the ones most
+worth analyzing — to be invisible to all downstream engines
+(`complexity`, `dead-code`, `smell`, `entrypoints`). On
+Wolfvin/Regrets ~40% of the codebase was silently dropped, including
+the worst hotspots (`scripts/validate.js` 2730 lines,
+`scripts/validate.py` 2361 lines).
+
+**Root cause investigation:** the tree-sitter 0.26 Python binding has
+a nondeterministic SIGSEGV on large files. Existing mitigations
+(`BaseParser._last_tree`, `parse_tree()`, `_gc.disable()`) reduce
+crash frequency but do not eliminate it. Verified by stress-testing
+synthetic and real-world files: crashes begin at ~250 lines for JS
+and ~500 lines for Python. The bug cannot be fixed from Python — it
+requires a binding upgrade.
+
+**Fix (issue #163):** instead of silently skipping, large files now
+use the REGEX FALLBACK parser (`parse_js_backend_fallback` /
+`parse_python_fallback`), which gives partial coverage (function
+declarations + direct calls) instead of zero coverage. The result
+includes a `skipped_from_tree_sitter` field so callers know
+tree-sitter was not used and why. The scan command aggregates all
+such entries into a top-level `skipped_from_tree_sitter` list in its
+JSON output.
+
+Additional hardening applied to `JSBackendParser`:
+- Iterative DFS walk replaces recursive `_walk` (prevents Python
+  stack frames from holding stale Node references across function
+  boundaries — reduces crash frequency on smaller files).
+- Iterative DFS in `_find_calls_in_scope` for the same reason.
+
+Thresholds raised: JS 100 → 250, Python 200 → 500 (largest values
+that passed 5 consecutive stress-test runs without SIGSEGV).
+
+`outline_engine.py` similarly no longer preemptively falls back to
+regex for JS files > 100 lines or Python files > 200 lines — it
+attempts tree-sitter first and only falls back on actual parse
+exception.
+
 ### LSP Status Entry-Point Unification (issue #33)
 
 The `codelens --lsp-status` top-level flag (intercepted in

scripts/commands/scan.py

@@ -797,11 +797,16 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list]
                     refs = js_be_parser.extract_references(content, rel_path)
                 else:
                     refs = parse_js_backend_fallback(content, rel_path)
-                js_backend_data.append({
+                item = {
                     "path": rel_path,
                     "nodes": refs.get("nodes", []),
                     "edges": refs.get("edges", [])
-                })
+                }
+                # Issue #163: surface files that fell back to regex
+                # due to tree-sitter binding segfault risk.
+                if refs.get("skipped_from_tree_sitter"):
+                    item["skipped_from_tree_sitter"] = refs["skipped_from_tree_sitter"]
+                js_backend_data.append(item)
             except IOError:
                 logger.debug(f"Failed to read JS backend file: {path}")
 
@@ -853,11 +858,16 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list]
                     refs = py_parser.extract_references(content, os.path.relpath(path, workspace))
                 else:
                     refs = parse_python_fallback(content, os.path.relpath(path, workspace))
-                python_data.append({
+                item = {
                     "path": os.path.relpath(path, workspace),
                     "nodes": refs.get("nodes", []),
                     "edges": refs.get("edges", [])
-                })
+                }
+                # Issue #163: surface files that fell back to regex
+                # due to tree-sitter binding segfault risk.
+                if refs.get("skipped_from_tree_sitter"):
+                    item["skipped_from_tree_sitter"] = refs["skipped_from_tree_sitter"]
+                python_data.append(item)
             except IOError:
                 logger.debug(f"Failed to read Python file: {path}")
 
@@ -1352,6 +1362,16 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list]
     except Exception:
         logger.debug("final graph_stats query failed", exc_info=True)
 
+    # Issue #163: aggregate files that fell back to regex because
+    # tree-sitter 0.26 has a nondeterministic SIGSEGV on large files
+    # (issue #116). Surface them in the scan output so callers know
+    # coverage is partial — silent skip was the original bug.
+    _skipped_from_tree_sitter = []
+    for _item in (js_backend_data + python_data + rust_data):
+        _skip = _item.get("skipped_from_tree_sitter")
+        if _skip:
+            _skipped_from_tree_sitter.append(_skip)
+
     result = {
         "status": "ok",
         "workspace": workspace,
@@ -1439,6 +1459,12 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list]
                 "skip_percent": 0.0, "elapsed_sec": 0.0,
             },
         },
+        # Issue #163: files that used regex fallback instead of
+        # tree-sitter due to the binding segfault risk on large files
+        # (issue #116). Each entry has: file, lines, threshold, reason,
+        # fallback_used. Empty list means all files were parsed with
+        # tree-sitter (full AST accuracy).
+        "skipped_from_tree_sitter": _skipped_from_tree_sitter,
     }
 
     # Issue #56: print prefilter stats to stderr when --verbose. Matches

scripts/outline_engine.py

@@ -208,14 +208,13 @@ def _outline_javascript(content: str, detail: str) -> Dict:
     """Outline for JavaScript files."""
     outline = {"imports": [], "functions": [], "classes": [], "exports": [], "variables": []}
 
-    # Workaround for issue #116: tree-sitter-javascript 0.25 segfaults
-    # on files with deeply nested callbacks. Use regex fallback for
-    # files over 100 lines.
-    line_count = content.count('\n') + 1
-    if line_count > 100:
-        _extract_js_outline_regex(content, outline, detail)
-        return outline
-
+    # Issue #163: previously we preemptively fell back to regex for any
+    # JS file > 100 lines, which silently downgraded outline quality on
+    # the largest files in the repo. The tree-sitter GC issues that
+    # motivated that workaround (issue #116) are now mitigated in
+    # ``base_parser.py`` via ``_last_tree`` + iterative ``walk_tree``
+    # with GC disabled, so we attempt tree-sitter on every file and
+    # only fall back to regex on an actual parse exception.
     try:
         from grammar_loader import get_grammar_loader
         loader = get_grammar_loader()
@@ -305,13 +304,13 @@ def _outline_python(content: str, detail: str) -> Dict:
     """Outline for Python files."""
     outline = {"imports": [], "functions": [], "classes": [], "variables": []}
 
-    # Workaround for issue #116: tree-sitter-python 0.25 segfaults on
-    # large files. Use regex fallback for files over 200 lines.
-    line_count = content.count('\n') + 1
-    if line_count > 200:
-        _extract_python_outline_regex(content, outline, detail)
-        return outline
-
+    # Issue #163: previously we preemptively fell back to regex for any
+    # Python file > 200 lines, which silently downgraded outline quality
+    # on the largest files in the repo. The tree-sitter GC issues that
+    # motivated that workaround (issue #116) are now mitigated in
+    # ``base_parser.py`` via ``_last_tree`` + iterative ``walk_tree``
+    # with GC disabled, so we attempt tree-sitter on every file and
+    # only fall back to regex on an actual parse exception.
     try:
         from grammar_loader import get_grammar_loader
         loader = get_grammar_loader()