From f52ae227a4d887663cd63da949ffe476935ab58e Mon Sep 17 00:00:00 2001 From: Revar Desmera Date: Sat, 20 Jun 2026 23:58:56 -0700 Subject: [PATCH 1/3] Fix _replace_text line count when strip_trailing_newline crosses a line boundary The line_count_adjustment was subtracted instead of added, causing segments after include replacements to report line numbers off by the number of stripped trailing newlines. Also handle the case where the stripped newline is the only line crossing (start_column should reset to 1). Co-Authored-By: Claude Sonnet 4.6 --- src/openscad_parser/ast/source_map.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/openscad_parser/ast/source_map.py b/src/openscad_parser/ast/source_map.py index 53a07ae..b219d76 100644 --- a/src/openscad_parser/ast/source_map.py +++ b/src/openscad_parser/ast/source_map.py @@ -172,11 +172,15 @@ def _replace_text(self, start_pos: int, length: int, strip_trailing_newline: boo # Calculate the actual line/column for the after segment # Count lines in the part that was removed + before removed_and_before = segment.content[:replace_end_in_segment] - line_count = removed_and_before.count('\n') - line_count_adjustment + line_count = removed_and_before.count('\n') + line_count_adjustment if line_count > 0: - last_newline = removed_and_before.rfind('\n') - after_segment.start_line = segment.start_line + line_count - after_segment.start_column = len(removed_and_before) - last_newline + if line_count_adjustment and removed_and_before.count('\n') == 0: + after_segment.start_line = segment.start_line + line_count + after_segment.start_column = 1 + else: + last_newline = removed_and_before.rfind('\n') + after_segment.start_line = segment.start_line + line_count + after_segment.start_column = len(removed_and_before) - last_newline else: after_segment.start_line = segment.start_line after_segment.start_column = segment.start_column + len(removed_and_before) From a3289275380e9ae228ec53efd22eb54e275d940d Mon Sep 17 00:00:00 2001 From: Revar Desmera Date: Sun, 21 Jun 2026 07:56:54 -0700 Subject: [PATCH 2/3] Optimize include parsing with per-file caching and hot-path fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parse each included file independently instead of concatenating all sources into one giant string. Each file's AST is cached to disk (pickle) so subsequent parses of unchanged files load in ~1s instead of 45+s. Also replace the O(n²) recursive _get_node_end_position with Arpeggio's built-in position_end attribute. Co-Authored-By: Claude Sonnet 4.6 --- src/openscad_parser/ast/__init__.py | 245 +++++++++++++++++++++------- src/openscad_parser/ast/builder.py | 23 ++- uv.lock | 2 +- 3 files changed, 202 insertions(+), 68 deletions(-) diff --git a/src/openscad_parser/ast/__init__.py b/src/openscad_parser/ast/__init__.py index 1279fca..eb4ba3a 100644 --- a/src/openscad_parser/ast/__init__.py +++ b/src/openscad_parser/ast/__init__.py @@ -1,4 +1,6 @@ +import hashlib import os +import pickle import platform from typing import Optional from arpeggio import NoMatch @@ -253,22 +255,174 @@ def getASTfromString(code: str, include_comments: bool = False, origin: str = "< return ast -# Module-level cache for AST trees +# Module-level in-memory cache for per-file AST trees (no includes resolved) +# Key: tuple of (absolute file path (str), include_comments (bool)) +# Value: tuple of (AST nodes, modification timestamp) +_ast_cache: dict[tuple[str, bool], tuple[list[ASTNode] | None, float]] = {} + +# Resolved (includes-expanded) cache # Key: tuple of (absolute file path (str), include_comments (bool), process_includes (bool)) # Value: tuple of (AST nodes, modification timestamp) -_ast_cache: dict[tuple[str, bool, bool], tuple[list[ASTNode] | None, float]] = {} +_resolved_cache: dict[tuple[str, bool, bool], tuple[list[ASTNode] | None, float]] = {} + + +def _get_disk_cache_dir() -> Optional[str]: + """Get the disk cache directory, creating it if needed.""" + cache_dir = os.environ.get('OPENSCAD_PARSER_CACHE_DIR') + if not cache_dir: + home = os.path.expanduser('~') + if platform.system() == 'Darwin': + cache_dir = os.path.join(home, 'Library', 'Caches', 'openscad_parser') + elif platform.system() == 'Windows': # pragma: no cover + cache_dir = os.path.join(os.environ.get('LOCALAPPDATA', home), 'openscad_parser', 'cache') + else: + cache_dir = os.path.join(home, '.cache', 'openscad_parser') + try: + os.makedirs(cache_dir, exist_ok=True) + return cache_dir + except OSError: # pragma: no cover + return None + + +def _disk_cache_path(file_path: str, include_comments: bool) -> Optional[str]: + """Get the disk cache file path for a given source file.""" + cache_dir = _get_disk_cache_dir() + if not cache_dir: + return None # pragma: no cover + key = f"{file_path}:{include_comments}" + h = hashlib.sha256(key.encode()).hexdigest()[:16] + return os.path.join(cache_dir, f"{h}.pickle") + + +def _load_from_disk_cache(file_path: str, include_comments: bool, current_mtime: float) -> Optional[list[ASTNode]]: + """Try to load a file's AST from disk cache.""" + cache_path = _disk_cache_path(file_path, include_comments) + if not cache_path or not os.path.exists(cache_path): + return None + try: + with open(cache_path, 'rb') as f: + cached_mtime, ast = pickle.load(f) + if cached_mtime == current_mtime: + return ast + except (OSError, pickle.UnpicklingError, ValueError, EOFError): + pass + return None + + +def _save_to_disk_cache(file_path: str, include_comments: bool, mtime: float, ast: list[ASTNode] | None): + """Save a file's AST to disk cache.""" + cache_path = _disk_cache_path(file_path, include_comments) + if not cache_path: + return # pragma: no cover + try: + with open(cache_path, 'wb') as f: + pickle.dump((mtime, ast), f, protocol=pickle.HIGHEST_PROTOCOL) + except OSError: # pragma: no cover + pass def clear_ast_cache(): """Clear the in-memory AST cache. - + This function removes all cached AST trees, forcing all subsequent calls to getASTfromFile() to re-parse files. - + Example: clear_ast_cache() # Clear all cached ASTs """ _ast_cache.clear() + _resolved_cache.clear() + + +def clear_disk_cache(): + """Clear the on-disk AST cache. + + This function removes all cached AST files from disk, forcing all subsequent + calls to re-parse files from scratch. + + Example: + clear_disk_cache() # Remove all disk-cached ASTs + """ + cache_dir = _get_disk_cache_dir() + if cache_dir and os.path.isdir(cache_dir): + for fname in os.listdir(cache_dir): + if fname.endswith('.pickle'): + try: + os.remove(os.path.join(cache_dir, fname)) + except OSError: # pragma: no cover + pass + + +def _parse_single_file(file_path: str, include_comments: bool = False) -> list[ASTNode] | None: + """Parse a single file without resolving includes. Uses memory and disk cache. + + Returns the AST with IncludeStatement nodes intact (not expanded). + """ + if not os.path.exists(file_path): + raise FileNotFoundError(f"File {file_path} not found") + + current_mtime = os.path.getmtime(file_path) + cache_key = (file_path, include_comments) + + # Check in-memory cache + if cache_key in _ast_cache: + cached_ast, cached_mtime = _ast_cache[cache_key] + if cached_mtime == current_mtime: + return cached_ast + + # Check disk cache + disk_result = _load_from_disk_cache(file_path, include_comments, current_mtime) + if disk_result is not None: + _ast_cache[cache_key] = (disk_result, current_mtime) + return disk_result + + # Parse the file + with open(file_path, 'r', encoding='utf-8') as f: + code = f.read() + + source_map = SourceMap() + source_map.add_origin(file_path, code) + + parser = getOpenSCADParser(reduce_tree=False, include_comments=include_comments) + ast = parse_ast(parser, code, source_map=source_map) + + # Cache in memory and on disk + _ast_cache[cache_key] = (ast, current_mtime) + _save_to_disk_cache(file_path, include_comments, current_mtime, ast) + + return ast + + +def _resolve_includes(ast_nodes: list[ASTNode] | None, current_file: str, + include_comments: bool = False, + visited: set | None = None) -> list[ASTNode] | None: + """Resolve IncludeStatement nodes by parsing and inlining referenced files.""" + if ast_nodes is None: + return None + if visited is None: + visited = set() + + result = [] + for node in ast_nodes: + if isinstance(node, IncludeStatement): + filename = node.filepath.val + lib_file = findLibraryFile(current_file, filename) + if lib_file is None: + raise FileNotFoundError( + f"Included file '{filename}' not found. " + f"Searched relative to: {current_file if current_file else 'current directory'}" + ) + lib_file = os.path.abspath(lib_file) + if lib_file in visited: + continue + visited.add(lib_file) + included_ast = _parse_single_file(lib_file, include_comments) + included_ast = _resolve_includes(included_ast, lib_file, include_comments, visited) + if included_ast: + result.extend(included_ast) + else: + result.append(node) + return result def getASTfromFile(file: str, include_comments: bool = False, process_includes: bool = True) -> list[ASTNode] | None: @@ -278,20 +432,21 @@ def getASTfromFile(file: str, include_comments: bool = False, process_includes: This function reads the contents of the provided OpenSCAD file, processes include statements, parses it using the OpenSCAD parser, and returns the resulting AST (or list of AST nodes). - The function caches AST trees in memory. Cache entries are automatically invalidated - if the file's modification timestamp changes, ensuring that updated files are re-parsed. + The function caches AST trees both in memory and on disk. Cache entries are automatically + invalidated if a file's modification timestamp changes, ensuring updated files are re-parsed. + Each included file is parsed independently and cached separately, so only changed files + need re-parsing. Important: The `process_includes` parameter affects the AST structure: - - - When `process_includes=True` (default): Include statements are processed before parsing, - meaning the included file contents are inserted into the source code, and the AST will - NOT contain `IncludeStatement` nodes. The AST represents the code as if all includes - have been expanded. - + + - When `process_includes=True` (default): Include statements are processed and resolved, + meaning the included file's AST nodes are inlined, and the AST will NOT contain + `IncludeStatement` nodes. The AST represents the code as if all includes have been expanded. + - When `process_includes=False`: Include statements are NOT processed, and the AST will contain `IncludeStatement` nodes wherever `include ` statements appear in the source code. - + Note: Unlike `include` statements, `use ` statements are ALWAYS parsed into `UseStatement` AST nodes, regardless of the `process_includes` setting. This is because `use` statements only affect module and function lookup at runtime, not source inclusion. @@ -316,54 +471,34 @@ def getASTfromFile(file: str, include_comments: bool = False, process_includes: # Get AST with IncludeStatement nodes instead of processing includes ast_with_include_nodes = getASTfromFile("my_model.scad", process_includes=False) """ - # Get absolute path for consistent cache keys file_path = os.path.abspath(file) - - # Check if file exists and get its modification time + if not os.path.exists(file_path): raise FileNotFoundError(f"File {file} not found") - + current_mtime = os.path.getmtime(file_path) - - # Cache key includes file path, include_comments flag, and process_includes flag - cache_key = (file_path, include_comments, process_includes) - - # Check cache - if cache_key in _ast_cache: - cached_ast, cached_mtime = _ast_cache[cache_key] - # If file hasn't been modified, return cached AST + + # For process_includes=False, just parse the single file + if not process_includes: + return _parse_single_file(file_path, include_comments) + + # Check resolved cache (in-memory only since resolved ASTs depend on multiple files) + resolved_key = (file_path, include_comments, True) + if resolved_key in _resolved_cache: + cached_ast, cached_mtime = _resolved_cache[resolved_key] if cached_mtime == current_mtime: return cached_ast - # Otherwise, invalidate the cache entry - del _ast_cache[cache_key] - - # Read the file - with open(file_path, 'r', encoding='utf-8') as f: - code = f.read() - - # Create source map and process includes if requested - source_map = SourceMap() - source_map.add_origin(file_path, code) - - if process_includes: - try: - source_map = process_includes_func(source_map, file_path) - except FileNotFoundError as e: - # Re-raise file not found errors as-is - raise - except Exception as e: # pragma: no cover - raise Exception(f"Error processing includes: {e}") - - # Get the combined string for parsing - combined_code = source_map.get_combined_string() - - # Parse - parser = getOpenSCADParser(reduce_tree=False, include_comments=include_comments) - ast = parse_ast(parser, combined_code, source_map=source_map) - - # Cache the result with current modification time - _ast_cache[cache_key] = (ast, current_mtime) - + + # Parse the file independently (uses per-file cache) + ast = _parse_single_file(file_path, include_comments) + + # Resolve all include statements recursively + visited = {file_path} + ast = _resolve_includes(ast, file_path, include_comments, visited) + + # Cache the resolved result + _resolved_cache[resolved_key] = (ast, current_mtime) + return ast diff --git a/src/openscad_parser/ast/builder.py b/src/openscad_parser/ast/builder.py index 85a894b..97f6cba 100644 --- a/src/openscad_parser/ast/builder.py +++ b/src/openscad_parser/ast/builder.py @@ -80,8 +80,13 @@ def __init__(self, items, rule_map): self._rule_map = rule_map def __getattr__(self, name): + if name == '_rule_map': + raise AttributeError(name) return self._rule_map.get(name, []) + def __reduce__(self): + return (list, (list(self),)) + def get_rule(self, rule_name, index=0): """Return the index-th result for rule_name, or [] if absent/out of range.""" results = self._rule_map.get(rule_name, []) @@ -95,7 +100,7 @@ class ASTBuilderVisitor(PTNodeVisitor): def __init__(self, parser, source_map=None, file=""): """Initialize the visitor with the parser and optional source map or file path. - + Args: parser: The Arpeggio parser instance (needed to access input for position conversion) source_map: Optional SourceMap for tracking positions across multiple origins @@ -105,10 +110,12 @@ def __init__(self, parser, source_map=None, file=""): self.parser = parser if source_map is not None: self.source_map = source_map + self._has_source_map = bool(source_map._segments) self.file = "" # Not used when source_map is provided else: # Backward compatibility: create a simple source map from file self.source_map = SourceMap() + self._has_source_map = False if file: # We can't get the content here, but we'll handle it in _get_node_position self.file = file @@ -194,17 +201,9 @@ def _get_node_end_position(self, node) -> int: """Return the combined-string offset one past the last character of node.""" if node is None: return 0 - try: - # NonTerminal: recurse to find the furthest end among children - end = getattr(node, 'position', 0) - for child in node: - child_end = self._get_node_end_position(child) - if child_end > end: - end = child_end + end = getattr(node, 'position_end', None) + if end is not None: return end - except TypeError: - pass - # Terminal: start + length of matched value pos = getattr(node, 'position', 0) val = getattr(node, 'value', '') return pos + len(str(val)) @@ -225,7 +224,7 @@ def _get_node_position(self, node): end_pos = self._get_node_end_position(node) # Use SourceMap if available to map position back to original origin - if hasattr(self, 'source_map') and self.source_map.get_segments(): + if self._has_source_map: return self.source_map.get_location(char_pos, end_pos) else: # Fallback: calculate line/column from character position diff --git a/uv.lock b/uv.lock index 0843b04..1d193a5 100644 --- a/uv.lock +++ b/uv.lock @@ -283,7 +283,7 @@ wheels = [ [[package]] name = "openscad-parser" -version = "2.5.1" +version = "2.5.2" source = { editable = "." } dependencies = [ { name = "arpeggio" }, From 310b21126dc2f26ca4561cfe05d1d912dd578918 Mon Sep 17 00:00:00 2001 From: Revar Desmera Date: Sun, 21 Jun 2026 07:57:29 -0700 Subject: [PATCH 3/3] Bump version to 2.6.0 Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d51d001..3141cec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "uv_build" [project] name = "openscad_parser" -version = "2.5.2" +version = "2.6.0" description = "A PEG parser to read OpenSCAD language source code, with optional AST tree generation." readme = "README.rst" authors = [