diff --git a/scripts/commands/scan.py b/scripts/commands/scan.py index 46ad7880..11d40c78 100644 --- a/scripts/commands/scan.py +++ b/scripts/commands/scan.py @@ -768,6 +768,12 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list] # Parse JS Backend files js_backend_data = tsx_backend_data.copy() + # Issue #163: collect files explicitly skipped by parsers (e.g. files + # above the absolute hard limit of 10,000 lines). Each parser may + # return a ``skipped`` list in its refs dict — we aggregate them here + # so the scan result can report incomplete coverage to the caller + # instead of silently dropping files. + skipped_files: list = [] if files["js_backend"]: js_be_parser = None try: @@ -802,6 +808,8 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list] "nodes": refs.get("nodes", []), "edges": refs.get("edges", []) }) + # Issue #163: aggregate explicit skip entries + skipped_files.extend(refs.get("skipped", [])) except IOError: logger.debug(f"Failed to read JS backend file: {path}") @@ -830,6 +838,8 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list] "nodes": refs.get("nodes", []), "edges": refs.get("edges", []) }) + # Issue #163: aggregate explicit skip entries + skipped_files.extend(refs.get("skipped", [])) except IOError: logger.debug(f"Failed to read Rust file: {path}") @@ -858,6 +868,8 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list] "nodes": refs.get("nodes", []), "edges": refs.get("edges", []) }) + # Issue #163: aggregate explicit skip entries + skipped_files.extend(refs.get("skipped", [])) except IOError: logger.debug(f"Failed to read Python file: {path}") @@ -1395,6 +1407,12 @@ def cmd_scan(workspace: str, incremental: bool = False, plugins: Optional[list] "elixir_parsed": len(elixir_data), "dart_parsed": len(dart_data), "swift_parsed": len(swift_data), + # Issue #163: files explicitly skipped by parsers (e.g. > 10,000 + # lines). Always present — empty list means no files were skipped + # and coverage is complete. Each entry: {"file": str, "reason": + # str, "lines": int}. Replaces the previous silent-skip behavior + # where large files would vanish from the graph without trace. + "skipped_files": skipped_files, "scala_parsed": len(scala_data), "shell_parsed": len(shell_data), "gdscript_parsed": len(gdscript_data), diff --git a/scripts/parsers/js_backend_parser.py b/scripts/parsers/js_backend_parser.py index ad57cd7e..2eb6820d 100755 --- a/scripts/parsers/js_backend_parser.py +++ b/scripts/parsers/js_backend_parser.py @@ -47,7 +47,11 @@ def extract_references(self, content: str, file_path: str) -> Dict[str, List[Dic Extract function nodes and edges from backend JS. Returns: - {"nodes": [...], "edges": [...]} + { + "nodes": [...], + "edges": [...], + "skipped": [{"file": str, "reason": str, "lines": int}] # empty in normal operation + } Single-pass recursive walk (issue #116): the previous two-pass design held ``body_node`` Node references across function @@ -58,31 +62,43 @@ def extract_references(self, content: str, file_path: str) -> Dict[str, List[Dic declaration's body immediately, and disables the cyclic GC to prevent mid-walk collection. - Files larger than ``MAX_SAFE_JS_LINES`` are skipped with a - warning — tree-sitter 0.25 + tree-sitter-javascript 0.25 has a - known segfault on deeply-nested JS callbacks (issue #116) that - cannot be fully mitigated from Python. Skipping large files is - a pragmatic workaround until the binding is upgraded. + Issue #163: the previous MAX_SAFE_JS_LINES=100 threshold was + removed because it silently skipped the most complex (and + therefore most analysis-worthy) files in real codebases — e.g. + ``scripts/regret.js`` at 2,731 lines was invisible to + ``complexity``. The actual root-cause fix for the issue #116 + segfault is the single-pass walk + ``gc.disable()`` + holding + the Tree reference via ``self.parse_tree()`` (all in place). + + Files above ``ABSOLUTE_HARD_LIMIT_LINES`` (10,000) are still + skipped — they are almost always minified/generated code. The + skip is now explicit: the file appears in the ``skipped[]`` list + with reason ``file_too_large`` so the caller knows coverage is + incomplete (issue #163 DoD: "silent skip replaced with explicit + skipped[] list"). """ import gc as _gc import logging _log = logging.getLogger("codelens") - # Workaround for issue #116: tree-sitter-javascript 0.25 has - # nondeterministic segfaults on JS files with deeply nested - # callbacks. Skip them rather than crash the whole scan. - # Threshold is conservative — files under 100 lines rarely have - # deeply nested callbacks that trigger the binding bug. - MAX_SAFE_JS_LINES = 100 + ABSOLUTE_HARD_LIMIT_LINES = 10_000 line_count = content.count('\n') + 1 - if line_count > MAX_SAFE_JS_LINES: + if line_count > ABSOLUTE_HARD_LIMIT_LINES: _log.warning( - "Skipping large JS file %s (%d lines > %d threshold) " - "due to tree-sitter segfault risk (issue #116). " - "File will not appear in the graph.", - file_path, line_count, MAX_SAFE_JS_LINES, + "Skipping extremely large JS file %s (%d lines > %d hard limit). " + "Use .codelensignore to exclude if this is minified code. " + "File recorded in skipped[] list (issue #163).", + file_path, line_count, ABSOLUTE_HARD_LIMIT_LINES, ) - return {"nodes": [], "edges": []} + return { + "nodes": [], + "edges": [], + "skipped": [{ + "file": file_path, + "reason": "file_too_large", + "lines": line_count, + }], + } _gc_was_enabled = _gc.isenabled() if _gc_was_enabled: @@ -99,16 +115,26 @@ def extract_references(self, content: str, file_path: str) -> Dict[str, List[Dic MAX_DEPTH = 200 - def _walk(node: Node, depth: int): - """Recursive walk — body Node references never leave - this function's frame, so they cannot dangle across - function-boundary crossings.""" + # Issue #116/#163: iterative DFS walk. The previous recursive + # form held Node references in Python frames across function + # boundaries; combined with tree-sitter 0.26's binding this + # caused SIGSEGV on large JS files with deeply-nested callback + # chains. The iterative form holds an explicit (node, depth) + # stack — Node references never cross a Python function call + # boundary, which avoids the binding's GC invalidation bug. + # ``keep_alive`` pins every Node we visit for the duration of + # the walk so reference counting can't free them mid-walk. + keep_alive: List[Node] = [root] + stack: List[Tuple[Node, int]] = [(root, 0)] + while stack: + node, depth = stack.pop() if depth > MAX_DEPTH: - return + continue # Detect export_statement wrapper and mark exported if node.type == 'export_statement': for child in node.children: + keep_alive.append(child) if child.type in ('function_declaration', 'generator_function_declaration'): self._parse_and_collect_calls( child, source, file_path, nodes, edges, @@ -121,6 +147,7 @@ def _walk(node: Node, depth: int): ) elif child.type == 'lexical_declaration': for subchild in child.children: + keep_alive.append(subchild) if subchild.type == 'variable_declarator': self._parse_and_collect_calls( subchild, source, file_path, nodes, edges, @@ -128,6 +155,7 @@ def _walk(node: Node, depth: int): ) elif child.type == 'default_export_clause': for subchild in node.children: + keep_alive.append(subchild) if subchild.type == 'class_declaration': self._parse_and_collect_calls( subchild, source, file_path, nodes, edges, @@ -138,7 +166,7 @@ def _walk(node: Node, depth: int): subchild, source, file_path, nodes, edges, exported=True, ) - return # Don't double-count by recursing inside export_statement + continue # Don't double-count by recursing inside export_statement if node.type == 'function_declaration' or node.type == 'generator_function_declaration': self._parse_and_collect_calls( @@ -167,12 +195,14 @@ def _walk(node: Node, depth: int): node, source, file_path, nodes, edges, ) - # Recurse into children to find nested declarations - for child in node.children: - _walk(child, depth + 1) + # Push children in reverse so they pop in source order + if depth + 1 <= MAX_DEPTH: + children = node.children + for child in reversed(children): + keep_alive.append(child) + stack.append((child, depth + 1)) - _walk(root, 0) - return {"nodes": nodes, "edges": edges} + return {"nodes": nodes, "edges": edges, "skipped": []} finally: if _gc_was_enabled: _gc.enable() @@ -437,10 +467,15 @@ def _find_calls_in_scope(self, body_node: Optional[Node], source: bytes, file_path: str) -> List[Dict]: """Find all function calls within a function body. - Uses a local recursive walk (issue #116) instead of - :meth:`self.walk_tree` so body Node references never cross a - function boundary — the body subtree is fully processed before - this method returns. + Issue #116/#163: iterative DFS walk. The previous recursive form + caused SIGSEGV on large JS files with deeply-nested callback + chains — the tree-sitter 0.26 binding invalidates Node pointers + when Python frames holding them are entered/exited repeatedly. + The iterative form holds an explicit (node, depth) stack and a + ``keep_alive`` list that pins every visited Node for the + duration of the walk so reference counting can't free them + mid-walk. Symmetric with the iterative walk used in + :meth:`extract_references`. """ if not body_node: return [] @@ -448,9 +483,12 @@ def _find_calls_in_scope(self, body_node: Optional[Node], source: bytes, calls: List[Dict] = [] MAX_DEPTH = 200 - def _walk_calls(node: Node, depth: int): + keep_alive: List[Node] = [body_node] + stack: List[Tuple[Node, int]] = [(body_node, 0)] + while stack: + node, depth = stack.pop() if depth > MAX_DEPTH: - return + continue if node.type == 'call_expression': call_info = self._parse_call(node, source) if call_info: @@ -459,10 +497,12 @@ def _walk_calls(node: Node, depth: int): call_info = self._parse_new_expression(node, source) if call_info: calls.append(call_info) - for child in node.children: - _walk_calls(child, depth + 1) + if depth + 1 <= MAX_DEPTH: + children = node.children + for child in reversed(children): + keep_alive.append(child) + stack.append((child, depth + 1)) - _walk_calls(body_node, 0) return calls def _parse_call(self, node: Node, source: bytes) -> Optional[Dict]: diff --git a/scripts/parsers/python_parser.py b/scripts/parsers/python_parser.py index e3dcbc9c..0af6ace8 100644 --- a/scripts/parsers/python_parser.py +++ b/scripts/parsers/python_parser.py @@ -20,7 +20,7 @@ - Same function name in multiple files → flag duplicate_define """ -from typing import Dict, List, Any, Optional +from typing import Dict, List, Any, Optional, Tuple try: from tree_sitter import Node @@ -71,32 +71,50 @@ def extract_references(self, content: str, file_path: str) -> Dict[str, List[Dic Returns: { "nodes": [{"id": str, "fn": str, "file": str, "line": int, "async": bool, "impl_for": str|None}], - "edges": [{"from": str, "to_fn": str}] + "edges": [{"from": str, "to_fn": str}], + "skipped": [{"file": str, "reason": str, "lines": int}] # empty list in normal operation } - Files larger than ``MAX_SAFE_PY_LINES`` are skipped with a - warning — tree-sitter-python 0.25 + tree-sitter 0.26 has a - known segfault on deeply-nested Python files (issue #116) that - cannot be fully mitigated from Python. + Issue #163: the previous MAX_SAFE_PY_LINES=200 threshold was removed + because it silently skipped the most complex (and therefore most + analysis-worthy) files in real codebases. The actual root-cause fix + for the issue #116 segfault is: + 1. ``BaseParser.parse`` stores the Tree on ``self._last_tree`` + so Node references stay valid for the lifetime of the parser + instance (already in place — see base_parser.py). + 2. ``gc.disable()`` is called for the duration of parse + walk + to prevent cyclic GC from invalidating tree-sitter Node + pointers mid-walk (already in place below). + + Files above ``ABSOLUTE_HARD_LIMIT_LINES`` (10,000) are still + skipped — they are almost always generated/minified code, not + human-written source. The skip is now explicit: the file appears + in the ``skipped[]`` list with reason ``file_too_large`` so the + caller knows coverage is incomplete (issue #163 DoD: "silent + skip replaced with explicit skipped[] list"). """ import gc as _gc import logging _log = logging.getLogger("codelens") - # Workaround for issue #116: tree-sitter-python 0.25 has - # nondeterministic segfaults on Python files with deeply nested - # functions/classes. Skip large files rather than crash the scan. - # 200 lines is conservative — small files rarely trigger the bug. - MAX_SAFE_PY_LINES = 200 + ABSOLUTE_HARD_LIMIT_LINES = 10_000 line_count = content.count('\n') + 1 - if line_count > MAX_SAFE_PY_LINES: + if line_count > ABSOLUTE_HARD_LIMIT_LINES: _log.warning( - "Skipping large Python file %s (%d lines > %d threshold) " - "due to tree-sitter segfault risk (issue #116). " - "File will not appear in the graph.", - file_path, line_count, MAX_SAFE_PY_LINES, + "Skipping extremely large Python file %s (%d lines > %d hard limit). " + "Use .codelensignore to exclude if this is generated code. " + "File recorded in skipped[] list (issue #163).", + file_path, line_count, ABSOLUTE_HARD_LIMIT_LINES, ) - return {"nodes": [], "edges": []} + return { + "nodes": [], + "edges": [], + "skipped": [{ + "file": file_path, + "reason": "file_too_large", + "lines": line_count, + }], + } # Disable cyclic GC during parse + walk to prevent tree-sitter # node invalidation (issue #116). See base_parser.walk_tree for @@ -105,40 +123,64 @@ def extract_references(self, content: str, file_path: str) -> Dict[str, List[Dic if _gc_was_enabled: _gc.disable() try: - return self._extract_references_impl(content, file_path) + result = self._extract_references_impl(content, file_path) finally: if _gc_was_enabled: _gc.enable() + # Forward-compat: always include skipped[] so callers can rely on + # the field being present (issue #163). + result.setdefault("skipped", []) + return result def _extract_references_impl(self, content: str, file_path: str) -> Dict[str, List[Dict[str, Any]]]: """Implementation of :meth:`extract_references` — called with GC disabled.""" - root = self.parse(content.encode('utf-8')) + # Issue #116/#163: use parse_tree and hold the Tree in a local + # variable for the entire walk. tree-sitter 0.26's binding does + # not make Node objects hold a strong reference to the Tree, so + # the Tree must be pinned explicitly — otherwise its refcount + # can drop to 0 mid-walk and Node pointers dangle. + tree_obj = self.parse_tree(content.encode('utf-8')) + root = tree_obj.root_node nodes = [] edges = [] - # Track declared functions and current class + # Track declared functions (informational — not used for walk control) declared_fns: Dict[str, str] = {} # fn_name → node_id - current_class: Optional[str] = None - current_fn_id: Optional[str] = None source = content.encode('utf-8') - def _walk(node, class_name=None, fn_id=None, depth=0, max_depth=200): - """Recursively walk the tree to find functions and calls. - - ``max_depth`` guards against pathological deeply-nested trees - that would otherwise overflow Python's recursion limit or - trigger the tree-sitter GC segfault (issue #116). 200 is well - above any real-world Python AST depth. - """ - nonlocal current_class, current_fn_id - if depth > max_depth: - return + MAX_DEPTH = 200 + + # Issue #116/#163: iterative DFS walk. The previous recursive form + # caused SIGSEGV on deeply-nested Python files (depth ≥ ~100) — + # the tree-sitter 0.26 binding invalidates Node pointers when + # Python frames holding them are entered/exited repeatedly. The + # iterative form holds an explicit (node, depth, class_name, + # fn_id) stack — Node references never cross a Python function + # call boundary, which avoids the binding's GC invalidation bug. + # ``keep_alive`` pins every Node we visit for the duration of the + # walk so reference counting can't free them mid-walk. Symmetric + # with the iterative walk used in JSBackendParser. + keep_alive: List[Any] = [root, tree_obj] + # Stack entries: (node, depth, class_name, fn_id) + # Push root's children directly so we don't waste a frame on root. + root_children = root.children + stack: List[Tuple[Any, int, Optional[str], Optional[str]]] = [ + (child, 0, None, None) for child in reversed(root_children) + ] + for child in root_children: + keep_alive.append(child) + + while stack: + node, depth, class_name, fn_id = stack.pop() + if depth > MAX_DEPTH: + continue if node.type == 'class_definition': # Track class context — also register the class itself as a node name_node = node.child_by_field_name('name') if name_node: + keep_alive.append(name_node) cls_name = self.get_text(name_node, source) line = self.get_line(node) @@ -148,15 +190,18 @@ def _walk(node, class_name=None, fn_id=None, depth=0, max_depth=200): superclasses_node = node.child_by_field_name('superclasses') superclass_names = [] if superclasses_node: + keep_alive.append(superclasses_node) for child in superclasses_node.children: + keep_alive.append(child) if child.type == 'identifier': superclass_names.append(self.get_text(child, source)) elif child.type == 'attribute': attr = child.child_by_field_name('attribute') if attr: + keep_alive.append(attr) superclass_names.append(self.get_text(attr, source)) - class_node = { + class_node_data = { "id": class_id, "fn": cls_name, "file": file_path, @@ -165,22 +210,25 @@ def _walk(node, class_name=None, fn_id=None, depth=0, max_depth=200): "type": "class", } if superclass_names: - class_node["superclasses"] = superclass_names + class_node_data["superclasses"] = superclass_names - nodes.append(class_node) + nodes.append(class_node_data) declared_fns[cls_name] = class_id - # Walk the body with class context + # Walk the body with class context — push children + # with the new class_name. body = node.child_by_field_name('body') if body: - for child in body.children: - _walk(child, class_name=cls_name, fn_id=fn_id, - depth=depth + 1, max_depth=max_depth) - return + keep_alive.append(body) + for child in reversed(body.children): + keep_alive.append(child) + stack.append((child, depth + 1, cls_name, fn_id)) + continue elif node.type == 'function_definition': name_node = node.child_by_field_name('name') if name_node: + keep_alive.append(name_node) fn_name = self.get_text(name_node, source) line = self.get_line(node) @@ -204,25 +252,25 @@ def _walk(node, class_name=None, fn_id=None, depth=0, max_depth=200): nodes.append(node_data) declared_fns[fn_name] = node_id - # Walk the body with function context + # Walk the body with function context — push children + # with the new fn_id. body = node.child_by_field_name('body') if body: - old_fn_id = current_fn_id - current_fn_id = node_id - for child in body.children: - _walk(child, class_name=class_name, fn_id=node_id, - depth=depth + 1, max_depth=max_depth) - current_fn_id = old_fn_id - return + keep_alive.append(body) + for child in reversed(body.children): + keep_alive.append(child) + stack.append((child, depth + 1, class_name, node_id)) + continue elif node.type == 'decorator': # Skip decorators - they reference functions but aren't calls - return + continue elif node.type == 'call' and fn_id: # Function call: name(args) or obj.method(args) func_node = node.child_by_field_name('function') if func_node: + keep_alive.append(func_node) call_name = self.get_text(func_node, source) # Handle attribute access: obj.method @@ -230,6 +278,8 @@ def _walk(node, class_name=None, fn_id=None, depth=0, max_depth=200): attr_node = func_node.child_by_field_name('attribute') obj_node = func_node.child_by_field_name('object') if attr_node and obj_node: + keep_alive.append(attr_node) + keep_alive.append(obj_node) method_name = self.get_text(attr_node, source) obj_name = self.get_text(obj_node, source) if method_name not in PYTHON_SKIP_NAMES: @@ -246,12 +296,11 @@ def _walk(node, class_name=None, fn_id=None, depth=0, max_depth=200): "to_fn": call_name }) - # Recurse into children - for child in node.children: - _walk(child, class_name=class_name, fn_id=fn_id, - depth=depth + 1, max_depth=max_depth) - - for child in root.children: - _walk(child) + # Recurse into children with same context + if depth + 1 <= MAX_DEPTH: + children = node.children + for child in reversed(children): + keep_alive.append(child) + stack.append((child, depth + 1, class_name, fn_id)) return {"nodes": nodes, "edges": edges} diff --git a/tests/test_large_file_parsing.py b/tests/test_large_file_parsing.py new file mode 100644 index 00000000..c22df3ae --- /dev/null +++ b/tests/test_large_file_parsing.py @@ -0,0 +1,300 @@ +""" +Tests for issue #163: file size threshold too conservative. + +Verifies that: +- Large Python files (>= 200 lines, the old threshold) are now parsed +- Large JS files (>= 100 lines, the old threshold) are now parsed +- The ``skipped`` field is always present in parser output +- Files above the absolute hard limit (10,000 lines) are explicitly + skipped with an entry in the ``skipped[]`` list (not silently) +- Deeply-nested Python files (depth >= 100) parse without SIGSEGV + (the original issue #116 segfault trigger) +""" + +import os +import sys +import pytest + +SCRIPT_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts") +sys.path.insert(0, SCRIPT_DIR) + + +# Detect tree-sitter availability once at module load — used by skipif markers. +def _tree_sitter_available() -> bool: + try: + import tree_sitter # noqa: F401 + from base_parser import BaseParser # noqa: F401 + return True + except ImportError: + return False + + +_TS_AVAILABLE = _tree_sitter_available() +_TS_SKIP_REASON = "tree-sitter or base_parser not installed" + + +# ─── Fixture generators ───────────────────────────────────────── + + +def _gen_large_python(lines: int) -> str: + """Generate a realistic large Python file — many classes/methods. + + Structure: N classes × M methods, shallow nesting (depth 3-4). + Matches real-world codebase patterns (not pathological depth). + """ + out = ['"""Large Python fixture for issue #163."""', '', 'import os', ''] + # Each list element is one line once joined with '\n'. + # len(out) approximates the line count (off by at most 1). + while len(out) < lines: + for c in range(10): + out.append(f'class Service{c}:') + out.append(f' """Service {c}."""') + for m in range(10): + out.append(f' def method_{m}(self, x, y):') + out.append(f' """Method {m}."""') + out.append(f' if x > y:') + out.append(f' return self.helper(x)') + out.append(f' return self.helper(y)') + out.append(f' def helper(self, v):') + out.append(f' return v * 2') + out.append('') + if len(out) >= lines: + break + return '\n'.join(out) + + +def _gen_large_js(lines: int) -> str: + """Generate a realistic large JS file — many functions with 2-level + callback nesting (matches real fetch().then().then() patterns).""" + out = ['// Large JS fixture for issue #163.', "'use strict';", ''] + i = 0 + while len(out) < lines: + out.append(f'function handler_{i}(req, res) {{') + out.append(f' return fetch("/api/{i}")') + out.append(f' .then(r => r.json())') + out.append(f' .then(d => res.json(d));') + out.append(f'}}') + i += 1 + return '\n'.join(out) + + +def _gen_deeply_nested_python(depth: int) -> str: + """Generate a Python file with deeply-nested function definitions. + + This is the original issue #116 segfault trigger: each nested + function adds a level of AST depth. + """ + out = ['"""Deeply nested Python fixture."""', 'import os', ''] + out.append('def outer():') + for d in range(depth): + indent = ' ' * (d + 1) + out.append(f'{indent}def level_{d}(x):') + out.append(f'{indent} y = x + {d}') + out.append(f'{indent} if y > 0:') + out.append(' ' * (depth + 2) + 'return y * 2') + for d in reversed(range(depth)): + indent = ' ' * (d + 1) + out.append(f'{indent} return level_{d}(y)') + out.append(f'{indent}return level_{d}(0)') + out.append('outer()') + return '\n'.join(out) + + +# ─── Python parser tests ──────────────────────────────────────── + + +class TestPythonParserLargeFiles: + """Issue #163: large Python files must be parsed, not silently skipped.""" + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_file_above_old_threshold_is_parsed(self): + """Files > 200 lines (old MAX_SAFE_PY_LINES) must now be parsed.""" + from parsers.python_parser import PythonParser + parser = PythonParser() + # 250 lines — just above the old 200-line threshold + content = _gen_large_python(250) + line_count = content.count('\n') + 1 + assert line_count > 200, f"fixture too small: {line_count} lines" + + result = parser.extract_references(content, "large.py") + + # Must NOT be skipped — nodes should be extracted + assert len(result["nodes"]) > 0, "large file was silently skipped" + assert len(result["edges"]) > 0, "no edges extracted from large file" + assert result["skipped"] == [], "file should not be in skipped[]" + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_file_3000_lines_parses_without_segfault(self): + """DoD: files up to 3,000 lines parse without SIGSEGV.""" + from parsers.python_parser import PythonParser + parser = PythonParser() + content = _gen_large_python(3000) + line_count = content.count('\n') + 1 + assert line_count >= 3000, f"fixture too small: {line_count} lines" + + result = parser.extract_references(content, "big.py") + + assert len(result["nodes"]) > 100, "expected many nodes from 3000-line file" + assert result["skipped"] == [] + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_deeply_nested_python_parses_without_segfault(self): + """Issue #116 root cause: deeply-nested Python must not segfault. + + The old threshold (200 lines) was added to avoid this segfault. + With the iterative walk + keep_alive fix, depth 100+ should work. + """ + from parsers.python_parser import PythonParser + parser = PythonParser() + content = _gen_deeply_nested_python(100) + line_count = content.count('\n') + 1 + + result = parser.extract_references(content, "nested.py") + + # 100 nested functions + outer = 101 nodes + assert len(result["nodes"]) >= 100, ( + f"expected >=100 nodes from depth-100 nesting, got {len(result['nodes'])}" + ) + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_skipped_field_always_present(self): + """The ``skipped`` field must always be present in the return value, + even when nothing is skipped (forward-compat for issue #163).""" + from parsers.python_parser import PythonParser + parser = PythonParser() + result = parser.extract_references("def f(): pass\n", "small.py") + assert "skipped" in result + assert result["skipped"] == [] + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_file_above_hard_limit_is_explicitly_skipped(self): + """Files > 10,000 lines are skipped with an explicit ``skipped[]`` + entry — not silently. Issue #163 DoD: 'silent skip replaced with + explicit skipped[] list'.""" + from parsers.python_parser import PythonParser + parser = PythonParser() + content = _gen_large_python(10050) + line_count = content.count('\n') + 1 + assert line_count > 10000, f"fixture too small: {line_count} lines" + + result = parser.extract_references(content, "huge.py") + + assert result["nodes"] == [], "huge file should not produce nodes" + assert len(result["skipped"]) == 1, ( + f"expected 1 skipped entry, got {len(result['skipped'])}" + ) + skip = result["skipped"][0] + assert skip["file"] == "huge.py" + assert skip["reason"] == "file_too_large" + assert skip["lines"] == line_count + + +# ─── JS backend parser tests ──────────────────────────────────── + + +class TestJSBackendParserLargeFiles: + """Issue #163: large JS files must be parsed, not silently skipped.""" + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_file_above_old_threshold_is_parsed(self): + """Files > 100 lines (old MAX_SAFE_JS_LINES) must now be parsed.""" + try: + from parsers.js_backend_parser import JSBackendParser + except (ImportError, RuntimeError) as e: + pytest.skip(f"JSBackendParser not available: {e}") + parser = JSBackendParser() + # 500 lines — well above the old 100-line threshold + content = _gen_large_js(500) + line_count = content.count('\n') + 1 + assert line_count > 100, f"fixture too small: {line_count} lines" + + result = parser.extract_references(content, "large.js") + + assert len(result["nodes"]) > 0, "large file was silently skipped" + assert len(result["edges"]) > 0, "no edges extracted from large file" + assert result["skipped"] == [], "file should not be in skipped[]" + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_file_3000_lines_parses_without_segfault(self): + """DoD: files up to 3,000 lines parse without SIGSEGV.""" + try: + from parsers.js_backend_parser import JSBackendParser + except (ImportError, RuntimeError) as e: + pytest.skip(f"JSBackendParser not available: {e}") + parser = JSBackendParser() + content = _gen_large_js(3000) + line_count = content.count('\n') + 1 + assert line_count >= 3000, f"fixture too small: {line_count} lines" + + result = parser.extract_references(content, "big.js") + + assert len(result["nodes"]) > 100, "expected many nodes from 3000-line file" + assert result["skipped"] == [] + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_skipped_field_always_present(self): + """The ``skipped`` field must always be present in the return value.""" + try: + from parsers.js_backend_parser import JSBackendParser + except (ImportError, RuntimeError) as e: + pytest.skip(f"JSBackendParser not available: {e}") + parser = JSBackendParser() + result = parser.extract_references( + "function f() { return 42; }", "small.js" + ) + assert "skipped" in result + assert result["skipped"] == [] + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_file_above_hard_limit_is_explicitly_skipped(self): + """Files > 10,000 lines are skipped with an explicit ``skipped[]`` + entry — not silently.""" + try: + from parsers.js_backend_parser import JSBackendParser + except (ImportError, RuntimeError) as e: + pytest.skip(f"JSBackendParser not available: {e}") + parser = JSBackendParser() + content = _gen_large_js(10050) + line_count = content.count('\n') + 1 + assert line_count > 10000, f"fixture too small: {line_count} lines" + + result = parser.extract_references(content, "huge.js") + + assert result["nodes"] == [], "huge file should not produce nodes" + assert len(result["skipped"]) == 1, ( + f"expected 1 skipped entry, got {len(result['skipped'])}" + ) + skip = result["skipped"][0] + assert skip["file"] == "huge.js" + assert skip["reason"] == "file_too_large" + assert skip["lines"] == line_count + + +# ─── Backward compat: existing small-file behavior unchanged ──── + + +class TestBackwardCompatSmallFiles: + """Small files (< 100 lines) must continue to parse exactly as before.""" + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_small_python_file_unchanged(self): + from parsers.python_parser import PythonParser + parser = PythonParser() + content = "def hello():\n return 42\n" + result = parser.extract_references(content, "small.py") + assert len(result["nodes"]) == 1 + assert result["nodes"][0]["fn"] == "hello" + assert result["skipped"] == [] + + @pytest.mark.skipif(not _TS_AVAILABLE, reason=_TS_SKIP_REASON) + def test_small_js_file_unchanged(self): + try: + from parsers.js_backend_parser import JSBackendParser + except (ImportError, RuntimeError) as e: + pytest.skip(f"JSBackendParser not available: {e}") + parser = JSBackendParser() + content = "function hello() { return 42; }" + result = parser.extract_references(content, "small.js") + assert len(result["nodes"]) == 1 + assert result["nodes"][0]["fn"] == "hello" + assert result["skipped"] == []