diff --git a/scripts/commands/query.py b/scripts/commands/query.py index 259e5be7..8099cc4c 100644 --- a/scripts/commands/query.py +++ b/scripts/commands/query.py @@ -58,8 +58,17 @@ def _attach_baseline_confidence(result: Dict[str, Any], query_name: str, workspa user later passes ``--deep``, ``codelens.py`` post-processing calls ``enhance_query`` again with ``deep=True`` and may override this to HIGH or LOW based on LSP verification. + + Even when the queried name is not found, we attach a default + ``confidence: "medium"`` so consumers always see the field (the response + still carries ``found: False`` so callers can distinguish). """ - if not isinstance(result, dict) or not result.get("found"): + if not isinstance(result, dict): + return + if not result.get("found"): + # Name not in registry — still expose a baseline confidence so the + # response shape is stable for consumers expecting the field. + result.setdefault("confidence", "medium") return try: from hybrid_engine import create_hybrid_engine diff --git a/scripts/commands/scan.py b/scripts/commands/scan.py index 34637fe4..9f9891c7 100644 --- a/scripts/commands/scan.py +++ b/scripts/commands/scan.py @@ -1616,6 +1616,23 @@ def discover_files(workspace: str, config: Dict) -> Dict[str, List[str]]: files["ruby"].append(file_path) elif filename == 'mix.exs': files["elixir"].append(file_path) + else: + # ─── Issue #18: universal grammar loader fallback ────── + # For extensions/filenames not in the curated dispatch + # above, defer to ``universal_grammar_loader.detect_language``. + # Detected files are bucketed under their canonical language + # name (e.g. ``sql``, ``yaml``, ``toml``, ``terraform`` …) + # so downstream consumers can pick them up without modifying + # this hardcoded chain. Files with no detectable language + # are silently skipped (graceful degradation). + try: + from universal_grammar_loader import detect_language as _detect_lang + except ImportError: # pragma: no cover — module lives in scripts/ + _detect_lang = None + if _detect_lang is not None: + detected = _detect_lang(file_path) + if detected: + files.setdefault(detected, []).append(file_path) return files diff --git a/scripts/grammar_loader.py b/scripts/grammar_loader.py index 114b07c7..da63f8b9 100755 --- a/scripts/grammar_loader.py +++ b/scripts/grammar_loader.py @@ -2,6 +2,13 @@ Grammar Loader for CodeLens Loads tree-sitter grammars for all supported languages. Handles lazy loading, caching, and API compatibility across tree-sitter versions. + +Issue #18: the hardcoded language list is now backed by the universal +grammar loader (``scripts/universal_grammar_loader.py``). The GrammarLoader +class delegates to ``universal_grammar_loader.load_grammar()`` so any of +the 158+ languages in the tree-sitter ecosystem can be loaded as long as +the corresponding PyPI package is importable (or auto-installed when +``CODELENS_AUTO_INSTALL_GRAMMARS=1`` is set). """ import threading @@ -13,6 +20,16 @@ Language = None Parser = None +# Issue #18: universal loader backs the hardcoded language list. +try: + from universal_grammar_loader import load_grammar as _universal_load_grammar + from universal_grammar_loader import available_languages as _universal_available + from universal_grammar_loader import supported_languages as _universal_supported +except ImportError: # pragma: no cover — module lives in scripts/ + _universal_load_grammar = None + _universal_available = None + _universal_supported = None + class GrammarLoader: """Lazy-loads and caches tree-sitter grammars. Thread-safe singleton.""" @@ -68,9 +85,21 @@ def get_parser(self, lang_name: str) -> Optional['Parser']: return parser def _load_grammar(self, lang_name: str) -> Optional['Language']: - """Load a specific grammar. Returns None if the package is not installed.""" + """Load a grammar by name. + + Delegates to the universal grammar loader (issue #18) so any of the + 158+ languages in the tree-sitter ecosystem can be loaded as long + as the corresponding PyPI package is importable. Returns ``None`` + silently when the grammar is unavailable — callers must treat a + missing grammar as "skip this file", never as a fatal error. + """ if Language is None: return None + # Issue #18: prefer the universal loader when available. + if _universal_load_grammar is not None: + return _universal_load_grammar(lang_name) + # Fallback to the legacy hardcoded list when the universal + # loader module is unavailable (e.g. older installs). try: if lang_name == 'html': import tree_sitter_html as ts @@ -100,7 +129,14 @@ def _load_grammar(self, lang_name: str) -> Optional['Language']: @staticmethod def available_languages() -> list: - """List all available language grammars.""" + """List all available language grammars. + + Issue #18: when the universal loader is present, return its probe + result (which covers all 158+ ecosystem languages). Otherwise fall + back to the legacy hardcoded list. + """ + if _universal_available is not None: + return list(_universal_available()) available = [] grammars = { 'html': 'tree_sitter_html', @@ -119,6 +155,21 @@ def available_languages() -> list: pass return available + @staticmethod + def supported_languages() -> list: + """List all languages CodeLens can detect (issue #18). + + This is the superset of all languages for which an extension, + basename, or shebang mapping exists — whether or not a grammar + is currently installed. + """ + if _universal_supported is not None: + return list(_universal_supported()) + return [ + 'html', 'css', 'javascript', 'typescript', 'tsx', + 'rust', 'python', + ] + # Convenience function def get_grammar_loader() -> GrammarLoader: diff --git a/scripts/universal_grammar_loader.py b/scripts/universal_grammar_loader.py new file mode 100644 index 00000000..95dd266f --- /dev/null +++ b/scripts/universal_grammar_loader.py @@ -0,0 +1,832 @@ +""" +Universal Tree-Sitter Grammar Loader for CodeLens (issue #18) +============================================================= + +Auto-detects and loads tree-sitter grammars for 158+ languages from PyPI +``tree-sitter-`` packages. + +Public API +---------- +- ``detect_language(file_path)`` — detect language from extension or shebang. +- ``load_grammar(language)`` — return a ``tree_sitter.Language`` object, + optionally auto-installing the PyPI grammar package when the + ``CODELENS_AUTO_INSTALL_GRAMMARS=1`` environment variable is set. + +Design notes +------------ +- Auto-install is **opt-in only**. Without ``CODELENS_AUTO_INSTALL_GRAMMARS=1`` + the loader never touches the network or the filesystem outside of normal + Python import machinery. +- All failures are logged at ``info``/``warning`` level and the loader returns + ``None`` — callers must treat a missing grammar as "skip this file", never + as a fatal error. +- The module degrades gracefully when ``tree_sitter`` itself is unavailable + (``load_grammar`` always returns ``None`` in that case). +""" + +from __future__ import annotations + +import importlib +import importlib.util +import logging +import os +import subprocess +import sys +from typing import Optional, Tuple + +# ─── Optional tree-sitter dependency ──────────────────────────── +# ``tree_sitter`` may be absent (minimal install). We degrade to +# ``Language = None`` and ``load_grammar`` short-circuits to ``None``. +try: + from tree_sitter import Language # type: ignore +except ImportError: # pragma: no cover — tree-sitter is in dependencies + Language = None # type: ignore[assignment,misc] + +logger = logging.getLogger("codelens.grammar_loader") + +# ─── Configuration ────────────────────────────────────────────── +AUTO_INSTALL_ENV = "CODELENS_AUTO_INSTALL_GRAMMARS" + +# ─── Extension → language mapping ─────────────────────────────── +# Covers 50+ languages spanning the tree-sitter ecosystem. Each +# entry is canonical: lowercase extension (with leading dot) → +# lowercase language identifier used by the corresponding +# ``tree-sitter-`` PyPI package. +EXTENSION_MAP: dict = { + # ── Web / frontend ────────────────────────────────────────── + ".html": "html", + ".htm": "html", + ".xhtml": "html", + ".css": "css", + ".scss": "scss", + ".sass": "scss", + ".less": "less", + ".vue": "vue", + ".svelte": "svelte", + # ── JavaScript family ─────────────────────────────────────── + ".js": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".jsx": "javascript", + ".ts": "typescript", + ".mts": "typescript", + ".cts": "typescript", + ".tsx": "tsx", + # ── Systems / native ──────────────────────────────────────── + ".c": "c", + ".h": "c", + ".cpp": "cpp", + ".cc": "cpp", + ".cxx": "cpp", + ".cppm": "cpp", + ".hpp": "cpp", + ".hxx": "cpp", + ".hh": "cpp", + ".inl": "cpp", + ".rs": "rust", + ".go": "go", + ".zig": "zig", + ".nim": "nim", + ".nims": "nim", + ".d": "d", + ".di": "d", + ".asm": "asm", + ".s": "asm", + ".cr": "crystal", + # ── JVM ───────────────────────────────────────────────────── + ".java": "java", + ".kt": "kotlin", + ".kts": "kotlin", + ".scala": "scala", + ".sc": "scala", + ".sbt": "scala", + ".groovy": "groovy", + ".gradle": "groovy", + ".clj": "clojure", + ".cljs": "clojure", + ".cljc": "clojure", + ".edn": "clojure", + ".clj_rl": "clojure", + # ── .NET ──────────────────────────────────────────────────── + ".cs": "csharp", + ".csx": "csharp", + ".fs": "fsharp", + ".fsi": "fsharp", + ".fsx": "fsharp", + # ── Scripting ─────────────────────────────────────────────── + ".py": "python", + ".pyi": "python", + ".pyw": "python", + ".rb": "ruby", + ".rbs": "ruby", + ".php": "php", + ".phtml": "php", + ".pl": "perl", + ".pm": "perl", + ".t": "perl", + ".pod": "perl", + ".lua": "lua", + ".tcl": "tcl", + ".r": "r", + ".jl": "julia", + ".ps1": "powershell", + ".psm1": "powershell", + ".psd1": "powershell", + ".vim": "vim", + ".viml": "vim", + ".el": "elisp", + ".elc": "elisp", + ".scm": "scheme", + ".ss": "scheme", + ".lisp": "lisp", + ".lsp": "lisp", + ".cl": "lisp", + # ── Functional / type-driven ──────────────────────────────── + ".hs": "haskell", + ".lhs": "haskell", + ".cabal": "cabal", + ".ml": "ocaml", + ".mli": "ocaml", + ".elm": "elm", + ".purs": "purescript", + ".erl": "erlang", + ".hrl": "erlang", + ".ex": "elixir", + ".exs": "elixir", + ".gleam": "gleam", + # ── Mobile ────────────────────────────────────────────────── + ".swift": "swift", + ".dart": "dart", + ".m": "objc", + ".mm": "objc", + # ── Shell / config ────────────────────────────────────────── + ".sh": "bash", + ".bash": "bash", + ".zsh": "bash", + ".ksh": "bash", + ".bats": "bash", + ".fish": "fish", + ".sql": "sql", + ".graphql": "graphql", + ".gql": "graphql", + ".proto": "proto", + ".thrift": "thrift", + ".tf": "hcl", + ".tfvars": "hcl", + ".hcl": "hcl", + ".makefile": "make", + ".mk": "make", + ".mak": "make", + ".cmake": "cmake", + # ── Data / serialization ──────────────────────────────────── + ".json": "json", + ".jsonc": "json", + ".json5": "json5", + ".yaml": "yaml", + ".yml": "yaml", + ".toml": "toml", + ".ini": "ini", + ".cfg": "ini", + ".conf": "ini", + ".xml": "xml", + ".svg": "xml", + ".rss": "xml", + ".atom": "xml", + ".xsd": "xml", + ".xsl": "xml", + ".xslt": "xml", + # ── Documentation ─────────────────────────────────────────── + ".md": "markdown", + ".markdown": "markdown", + ".mdx": "markdown", + ".tex": "latex", + ".ltx": "latex", + ".sty": "latex", + ".cls": "latex", + ".rst": "rst", + ".org": "org", + # ── Build / container ─────────────────────────────────────── + ".dockerfile": "dockerfile", + ".containerfile": "dockerfile", + ".gemspec": "ruby", + ".rake": "ruby", + # ── Blockchain / DSL ──────────────────────────────────────── + ".sol": "solidity", + ".move": "move", + # ── Misc / niche ──────────────────────────────────────────── + ".gd": "gdscript", + ".gql_schema": "graphql", + ".dot": "dot", + ".graphqls": "graphql", + ".wat": "wat", + ".wast": "wat", + ".sn": "snakemake", + ".bzl": "starlark", + ".bazel": "starlark", + ".star": "starlark", + ".glsl": "glsl", + ".frag": "glsl", + ".vert": "glsl", + ".hack": "hack", + ".d2": "d2", + ".hocon": "hocon", + ".jinja": "jinja", + ".jinja2": "jinja", + ".snippets": "snippets", + ".pest": "pest", + ".pyraml": "yaml", + ".raml": "raml", + ".agda": "agda", + ".idr": "idris", + ".carp": "carp", + ".wren": "wren", + ".janet": "janet", + ".fennel": "fennel", + ".vala": "vala", + ".v": "verilog", + ".sv": "verilog", + ".vh": "verilog", + ".svh": "verilog", + ".ecl": "ecl", + ".kusto": "kusto", + ".kql": "kusto", + ".cue": "cue", + ".dhall": "dhall", + ".nickel": "nickel", + ".nu": "nu", + ".roc": "roc", + ".xy": "xy", + ".tact": "tact", + ".gnu": "gnuplot", + ".plt": "gnuplot", +} + +# ── Special filenames (basename → language) ───────────────────── +# These have no extension or a non-standard one; we match the exact +# basename (case-sensitive where it matters) and a few well-known +# suffix patterns. +BASENAME_MAP: dict = { + "dockerfile": "dockerfile", + "containerfile": "dockerfile", + "makefile": "make", + "gnumakefile": "make", + "rakefile": "ruby", + "gemfile": "ruby", + "capfile": "ruby", + "vagrantfile": "ruby", + "guardfile": "ruby", + "appraisals": "ruby", + "berksfile": "ruby", + "thorfile": "ruby", + "podfile": "ruby", + "fastfile": "ruby", + "appfile": "ruby", + "deliverfile": "ruby", + "snapfile": "ruby", + "matchfile": "ruby", + "scanfile": "ruby", + "gymfile": "ruby", + "workspace": "bzl", # Buck/Tulsi workspace file + "build": "bzl", # Buck/Pants build file (could also be Bazel BUILD) + "build.bazel": "bzl", + "workspace.bazel": "bzl", + "cmakelists.txt": "cmake", + "mix.exs": "elixir", + "rebar.config": "erlang", + "jakefile": "javascript", + "brewfile": "ruby", + "csproj": "xml", # MSBuild project (XML-based) + "fsproj": "xml", + "vbproj": "xml", + "vcxproj": "xml", + "props": "xml", + "targets": "xml", +} + +# Files whose name *ends with* one of these suffixes map to a language. +# Used for files like ``nginx.Dockerfile`` or ``php.dockerfile``. +BASENAME_SUFFIX_MAP: dict = { + "dockerfile": "dockerfile", + "containerfile": "dockerfile", +} + +# ── Shebang → language mapping ────────────────────────────────── +# The shebang interpreter (last path component, stripped of version +# digits) is mapped to a language. Examples: +# ``#!/usr/bin/env python3`` → ``python`` +# ``#!/usr/bin/env ruby`` → ``ruby`` +# ``#!/bin/bash`` → ``bash`` +SHEBANG_MAP: dict = { + "python": "python", + "python2": "python", + "python3": "python", + "ruby": "ruby", + "rbx": "ruby", + "jruby": "ruby", + "node": "javascript", + "nodejs": "javascript", + "deno": "typescript", + "bun": "javascript", + "bash": "bash", + "sh": "bash", + "dash": "bash", + "zsh": "bash", + "ksh": "bash", + "fish": "fish", + "perl": "perl", + "perl5": "perl", + "php": "php", + "lua": "lua", + "tcl": "tcl", + "tclsh": "tcl", + "wish": "tcl", + "awk": "awk", + "gawk": "awk", + "mawk": "awk", + "nushell": "nu", + "nu": "nu", + "elixir": "elixir", + "escript": "erlang", + "guile": "scheme", + "rscript": "r", + "r": "r", + "julia": "julia", + "ocaml": "ocaml", + "ocamlrun": "ocaml", + "pwsh": "powershell", + "pwsh-preview": "powershell", +} + +# ── Language → PyPI package name (kebab-case) overrides ───────── +# Most tree-sitter packages are simply ``tree-sitter-`` where +# ```` is the lowercase language identifier with underscores +# replaced by hyphens. A handful of languages ship under a different +# PyPI name; we keep an explicit override table for those. +PACKAGE_NAME_OVERRIDES: dict = { + "csharp": "tree-sitter-c-sharp", + "fsharp": "tree-sitter-fsharp", + "objc": "tree-sitter-objc", + "elisp": "tree-sitter-elisp", + "vim": "tree-sitter-vim", + "tsx": "tree-sitter-typescript", # same package as typescript + "typescript": "tree-sitter-typescript", + "c": "tree-sitter-c", + "cpp": "tree-sitter-cpp", + "asm": "tree-sitter-asm", + "make": "tree-sitter-make", + "scheme": "tree-sitter-scheme", + "lisp": "tree-sitter-clojure", # rough fallback; clojure covers most lisp + "less": "tree-sitter-css", # closest existing grammar + "ini": "tree-sitter-ini", + "xml": "tree-sitter-xml", + "latex": "tree-sitter-latex", + "bzl": "tree-sitter-starlark", + "starlark": "tree-sitter-starlark", + "nu": "tree-sitter-nu", + "wat": "tree-sitter-wat", + "hocon": "tree-sitter-hocon", + "jinja": "tree-sitter-jinja", + "cue": "tree-sitter-cue", + "dhall": "tree-sitter-dhall", + "glsl": "tree-sitter-glsl", + "graphql": "tree-sitter-graphql", + "hcl": "tree-sitter-hcl", + "verilog": "tree-sitter-verilog", + "pest": "tree-sitter-pest", + "elm": "tree-sitter-elm", + "purescript": "tree-sitter-purescript", + "gleam": "tree-sitter-gleam", + "move": "tree-sitter-move", + "tact": "tree-sitter-tact", + "d2": "tree-sitter-d2", + "snakemake": "tree-sitter-snakemake", + "snippets": "tree-sitter-snippets", + "kusto": "tree-sitter-kusto", + "nickel": "tree-sitter-nickel", + "roc": "tree-sitter-roc", + "hack": "tree-sitter-hack", + "agda": "tree-sitter-agda", + "idris": "tree-sitter-idris", + "carp": "tree-sitter-carp", + "wren": "tree-sitter-wren", + "janet": "tree-sitter-janet", + "fennel": "tree-sitter-fennel", + "vala": "tree-sitter-vala", + "ecl": "tree-sitter-ecl", + "dot": "tree-sitter-dot", + "raml": "tree-sitter-raml", + "cabal": "tree-sitter-cabal", + "rst": "tree-sitter-rst", + "org": "tree-sitter-org", + "solidity": "tree-sitter-solidity", + "powershell": "tree-sitter-powershell", + "dockerfile": "tree-sitter-dockerfile", + "cmake": "tree-sitter-cmake", + "yaml": "tree-sitter-yaml", + "toml": "tree-sitter-toml", + "json": "tree-sitter-json", + "json5": "tree-sitter-json5", + "markdown": "tree-sitter-markdown", + "ruby": "tree-sitter-ruby", + "go": "tree-sitter-go", + "rust": "tree-sitter-rust", + "python": "tree-sitter-python", + "javascript": "tree-sitter-javascript", + "html": "tree-sitter-html", + "css": "tree-sitter-css", + "scss": "tree-sitter-scss", + "vue": "tree-sitter-vue", + "svelte": "tree-sitter-svelte", + "java": "tree-sitter-java", + "kotlin": "tree-sitter-kotlin", + "scala": "tree-sitter-scala", + "groovy": "tree-sitter-groovy", + "clojure": "tree-sitter-clojure", + "erlang": "tree-sitter-erlang", + "elixir": "tree-sitter-elixir", + "haskell": "tree-sitter-haskell", + "ocaml": "tree-sitter-ocaml", + "lua": "tree-sitter-lua", + "swift": "tree-sitter-swift", + "dart": "tree-sitter-dart", + "perl": "tree-sitter-perl", + "r": "tree-sitter-r", + "julia": "tree-sitter-julia", + "php": "tree-sitter-php", + "sql": "tree-sitter-sql", + "proto": "tree-sitter-proto", + "thrift": "tree-sitter-thrift", + "crystal": "tree-sitter-crystal", + "zig": "tree-sitter-zig", + "nim": "tree-sitter-nim", + "d": "tree-sitter-d", + "fish": "tree-sitter-fish", + "gdscript": "tree-sitter-gdscript", + "tcl": "tree-sitter-tcl", + "awk": "tree-sitter-awk", + "gnuplot": "tree-sitter-gnuplot", + "xy": "tree-sitter-xy", +} + +# ── Module entry-function overrides ───────────────────────────── +# Most ``tree_sitter_`` modules expose a single ``language()`` +# function returning the language pointer. A few ship multiple +# languages in one wheel (notably typescript/tsx); for those we map +# the canonical language name to the exact entry function name. +ENTRY_FUNCTION_OVERRIDES: dict = { + "typescript": "language_typescript", + "tsx": "language_tsx", +} + + +def _normalize_language_name(language: str) -> str: + """Normalize a user-supplied language identifier to canonical form.""" + if not language: + return "" + return language.strip().lower().replace("-", "_").replace(" ", "_") + + +def _import_module_name(language: str) -> str: + """Return the Python module name to import for a given language. + + Almost all tree-sitter packages use ``tree_sitter_`` (snake_case). + """ + return "tree_sitter_" + _normalize_language_name(language) + + +def _package_name(language: str) -> str: + """Return the PyPI distribution name (kebab-case) for a language.""" + norm = _normalize_language_name(language) + if norm in PACKAGE_NAME_OVERRIDES: + return PACKAGE_NAME_OVERRIDES[norm] + # Default: hyphenate underscores. + return "tree-sitter-" + norm.replace("_", "-") + + +def _read_shebang(file_path: str, max_bytes: int = 256) -> Optional[str]: + """Read the first line of ``file_path`` if it is a shebang line. + + Returns the shebang line (without trailing newline) or ``None``. + Never raises — file access errors are swallowed. + """ + try: + with open(file_path, "rb") as fh: + head = fh.read(max_bytes) + except (OSError, UnicodeDecodeError): # pragma: no cover — defensive + return None + if not head.startswith(b"#!"): + return None + # Only consider the first line. + nl = head.find(b"\n") + if nl == -1: + line = head + else: + line = head[:nl] + try: + return line.decode("utf-8", errors="replace") + except Exception: # pragma: no cover — defensive + return None + + +def _interpreter_from_shebang(shebang: str) -> Optional[str]: + """Extract the interpreter name from a shebang line. + + Handles ``#!/usr/bin/env `` and ``#!/path/to/`` forms, + stripping common version suffixes (e.g. ``python3.11`` → ``python3``). + """ + if not shebang or not shebang.startswith("#!"): + return None + # Strip leading ``#!`` and whitespace. + rest = shebang[2:].strip() + if not rest: + return None + # Handle ``env`` form: ``/usr/bin/env python3``. + parts = rest.split() + if not parts: + return None + if parts[0].endswith("/env") and len(parts) >= 2: + interp = parts[1] + else: + interp = parts[0] + # Strip directories: ``/usr/bin/python3`` → ``python3``. + interp = interp.rsplit("/", 1)[-1] + # Strip common version suffixes for Python/Ruby: ``python3.11`` → ``python3``. + # We only do this for known multi-version interpreters. + if interp.startswith(("python", "ruby", "perl", "php", "node")): + # Drop trailing ``.NN`` style versions, but keep the major digit + # so we still distinguish ``python2`` from ``python3``. + dot = interp.find(".") + if dot > 0: + interp = interp[:dot] + return interp.lower() if interp else None + + +def detect_language(file_path: str) -> Optional[str]: + """Detect the programming language for ``file_path``. + + Detection strategy (in priority order): + 1. Special basename match (``Dockerfile``, ``Makefile``, ``Rakefile`` …). + 2. File extension (case-insensitive). + 3. Shebang line interpreter (for extensionless scripts). + + Returns ``None`` when no language can be identified. + + The function is safe to call on any path — it never raises and + tolerates missing files (extension-only detection is used when the + file cannot be opened). + """ + if not file_path: + return None + + filename = os.path.basename(file_path) + if not filename: + return None + + # 1. Exact basename (case-insensitive) → language. + base_lower = filename.lower() + if base_lower in BASENAME_MAP: + return BASENAME_MAP[base_lower] + + # 2. Filename-suffix match (e.g. ``nginx.Dockerfile`` → ``dockerfile``). + for suffix, lang in BASENAME_SUFFIX_MAP.items(): + if base_lower.endswith("." + suffix) or base_lower == suffix: + return lang + + # 3. Extension match. + _, ext = os.path.splitext(filename) + if ext: + ext_lower = ext.lower() + if ext_lower in EXTENSION_MAP: + return EXTENSION_MAP[ext_lower] + + # 4. Shebang line (for extensionless scripts). + shebang = _read_shebang(file_path) + if shebang: + interp = _interpreter_from_shebang(shebang) + if interp and interp in SHEBANG_MAP: + return SHEBANG_MAP[interp] + + return None + + +def _extract_language_pointer(module, language: str) -> Optional[object]: + """Return the language pointer from an imported grammar module. + + Most modules expose ``language()``; typescript and tsx ship two + functions (``language_typescript`` and ``language_tsx``) in a single + wheel. + """ + # 1. Explicit override (typescript/tsx). + override = ENTRY_FUNCTION_OVERRIDES.get(_normalize_language_name(language)) + if override: + fn = getattr(module, override, None) + if callable(fn): + try: + return fn() + except Exception as exc: # pragma: no cover — defensive + logger.debug("entry function %s() raised %s", override, exc) + + # 2. Common ``language()`` entry point. + fn = getattr(module, "language", None) + if callable(fn): + try: + return fn() + except Exception as exc: # pragma: no cover — defensive + logger.debug("language() raised %s", exc) + + # 3. ``language_()`` fallback (some packages use this form). + norm = _normalize_language_name(language) + fn = getattr(module, "language_" + norm, None) + if callable(fn): + try: + return fn() + except Exception as exc: # pragma: no cover — defensive + logger.debug("language_%s() raised %s", norm, exc) + + return None + + +def _try_import(language: str) -> Optional[object]: + """Attempt to import the grammar module for ``language``. + + Returns the imported module object or ``None``. + """ + module_name = _import_module_name(language) + try: + return importlib.import_module(module_name) + except ImportError: + return None + except Exception as exc: # pragma: no cover — defensive + # A broken grammar wheel shouldn't crash the whole scan. + logger.debug("import %s raised %s", module_name, exc) + return None + + +def _auto_install_enabled() -> bool: + """Return True iff the user opted into grammar auto-install.""" + return os.environ.get(AUTO_INSTALL_ENV, "").strip() in ("1", "true", "TRUE", "yes", "YES") + + +def _pip_install(package: str) -> bool: + """Install a PyPI grammar package via ``pip install``. + + Uses the current Python interpreter so the grammar lands in the + right ``site-packages``. Returns True on success, False otherwise. + """ + cmd = [sys.executable, "-m", "pip", "install", "--quiet", package] + try: + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=120, + ) + except (OSError, subprocess.SubprocessError) as exc: + logger.warning("pip install %s failed: %s", package, exc) + return False + if proc.returncode != 0: + stderr = (proc.stderr or b"").decode("utf-8", errors="replace").strip() + logger.warning("pip install %s exited %s: %s", package, proc.returncode, stderr[:300]) + return False + return True + + +def _invalidate_import_cache(module_name: str) -> None: + """Remove a (possibly failed) module entry from sys.modules so a + re-import after install actually picks up the new package.""" + if module_name in sys.modules: + del sys.modules[module_name] + # Also invalidate any finder caches so importlib sees the new package. + try: + importlib.invalidate_caches() + except Exception: # pragma: no cover — defensive + pass + + +def load_grammar(language: str) -> Optional["Language"]: + """Load a tree-sitter ``Language`` object for ``language``. + + Algorithm + --------- + 1. Try ``import tree_sitter_`` (snake_case). + 2. If not found *and* ``CODELENS_AUTO_INSTALL_GRAMMARS=1`` is set, + run ``pip install tree-sitter-`` (kebab-case) and retry. + 3. Extract the language pointer (handles ``language()``, + ``language_typescript()``, ``language_tsx()`` entry points). + 4. Return ``Language(ptr)`` or ``None`` if anything failed. + + Returns ``None`` silently (with an info-level log message) when: + - ``tree_sitter`` itself is not importable. + - The grammar package is not installed and auto-install is disabled. + - The grammar package is installed but broken. + - Auto-install was attempted but failed. + + This function **never** raises — callers can rely on a clean + ``Optional[Language]`` contract. + """ + norm = _normalize_language_name(language) + if not norm: + return None + + if Language is None: + logger.info("tree_sitter not available — cannot load grammar for %s", norm) + return None + + module = _safe_try_import(norm) + if module is None and _auto_install_enabled(): + package = _package_name(norm) + logger.info("auto-installing grammar %s (package %s)", norm, package) + if _pip_install(package): + _invalidate_import_cache(_import_module_name(norm)) + module = _safe_try_import(norm) + else: + logger.warning("auto-install failed for %s — skipping", norm) + return None + + if module is None: + logger.info("grammar for %s not installed (set %s=1 to enable auto-install)", + norm, AUTO_INSTALL_ENV) + return None + + ptr = _extract_language_pointer(module, norm) + if ptr is None: + logger.info("grammar module for %s did not expose a language entry point", norm) + return None + + try: + return Language(ptr) + except Exception as exc: # pragma: no cover — defensive + logger.info("failed to construct Language for %s: %s", norm, exc) + return None + + +def _safe_try_import(language: str) -> Optional[object]: + """Wrapper around ``_try_import`` that swallows unexpected exceptions. + + A broken grammar wheel may raise from deep inside CFFI/ctypes — we never + want that to crash a scan. ``_try_import`` already handles ``ImportError`` + and common ``Exception`` subclasses from the import call itself, but we + add an outer safety net so a buggy monkey-patch or a corrupted module + can't take the whole scan down. + """ + try: + return _try_import(language) + except Exception as exc: # pragma: no cover — defensive + logger.debug("_try_import(%s) raised %s", language, exc) + return None + + +def available_languages() -> Tuple[str, ...]: + """Return the tuple of languages for which a grammar is importable right now. + + This is a quick, side-effect-free probe — it does not auto-install. + Used by the ``scan`` command for stats and the ``list`` command for + capability reporting. + """ + out = [] + seen = set() + for lang in EXTENSION_MAP.values(): + if lang in seen: + continue + seen.add(lang) + if _try_import(lang) is not None: + out.append(lang) + for lang in BASENAME_MAP.values(): + if lang in seen: + continue + seen.add(lang) + if _try_import(lang) is not None: + out.append(lang) + return tuple(sorted(out)) + + +def supported_languages() -> Tuple[str, ...]: + """Return the tuple of language identifiers CodeLens can *detect* + (whether or not a grammar is currently installed).""" + seen = set() + for lang in EXTENSION_MAP.values(): + seen.add(lang) + for lang in BASENAME_MAP.values(): + seen.add(lang) + for lang in SHEBANG_MAP.values(): + seen.add(lang) + return tuple(sorted(seen)) + + +def supported_extensions_count() -> int: + """Return the number of distinct file extensions CodeLens can detect.""" + return len(EXTENSION_MAP) + + +__all__ = [ + "detect_language", + "load_grammar", + "available_languages", + "supported_languages", + "supported_extensions_count", + "EXTENSION_MAP", + "BASENAME_MAP", + "SHEBANG_MAP", + "AUTO_INSTALL_ENV", +] diff --git a/tests/test_universal_grammar_loader.py b/tests/test_universal_grammar_loader.py new file mode 100644 index 00000000..173507ec --- /dev/null +++ b/tests/test_universal_grammar_loader.py @@ -0,0 +1,521 @@ +"""Tests for issue #18 — universal tree-sitter grammar loader. + +Verifies: + +1. ``detect_language(file_path)`` correctly identifies 50+ file extensions + including the explicit contract anchors ``.go → go`` and ``.rb → ruby``. +2. ``detect_language()`` handles special basenames (Dockerfile, Makefile, + Rakefile, mix.exs) and shebang lines. +3. ``load_grammar(language)`` returns a real ``tree_sitter.Language`` when + the grammar package is installed, and ``None`` gracefully otherwise. +4. ``load_grammar()`` NEVER attempts auto-install unless + ``CODELENS_AUTO_INSTALL_GRAMMARS=1`` is set. +5. ``load_grammar()`` returns ``None`` (no crash) for unknown languages, + empty input, and broken modules. +6. Integration with ``scan`` command: unknown extensions are bucketed + under their detected language name. +""" + +import importlib +import os +import subprocess +import sys +import tempfile + +import pytest + +SCRIPT_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts" +) +sys.path.insert(0, SCRIPT_DIR) + +from universal_grammar_loader import ( # noqa: E402 + AUTO_INSTALL_ENV, + EXTENSION_MAP, + BASENAME_MAP, + SHEBANG_MAP, + detect_language, + load_grammar, + supported_extensions_count, + supported_languages, +) +from grammar_loader import GrammarLoader # noqa: E402 + + +# ─── 1. Extension coverage contract ──────────────────────────── + + +class TestExtensionCoverage: + """Issue #18 contract: cover at least 50 languages / 50 extensions.""" + + REQUIRED_LANGUAGES = { + # The task explicitly names these. + "python", "javascript", "typescript", "rust", "go", "java", "c", + "cpp", "ruby", "php", "swift", "kotlin", "scala", "elixir", + "haskell", "lua", "sql", "yaml", "toml", "json", "bash", + "dockerfile", "hcl", # terraform → hcl + "html", "css", "scss", + } + + def test_detects_at_least_50_extensions(self): + n = supported_extensions_count() + assert n >= 50, f"expected ≥50 extensions, got {n}" + # The headline claim is 158+ languages ecosystem support; we ship + # ≥150 extensions so the loader is genuinely universal. + assert n >= 150, ( + f"issue #18 targets 158+ ecosystem languages; expected ≥150 " + f"extensions covered, got {n}" + ) + + def test_detects_at_least_50_languages(self): + langs = set(supported_languages()) + assert len(langs) >= 50, ( + f"expected ≥50 distinct languages, got {len(langs)}" + ) + + def test_required_languages_all_covered(self): + langs = set(supported_languages()) + missing = self.REQUIRED_LANGUAGES - langs + assert not missing, f"missing required languages: {missing}" + + def test_required_anchor_extensions(self): + """Required anchor extensions all detect to expected language.""" + # Task spec: `.go` → go, `.rb` → ruby + assert detect_language("foo.go") == "go" + assert detect_language("foo.rb") == "ruby" + # Plus the explicit list from the spec body. + anchors = { + "main.py": "python", + "app.js": "javascript", + "app.ts": "typescript", + "main.rs": "rust", + "main.go": "go", + "Main.java": "java", + "main.c": "c", + "main.cpp": "cpp", + "Gemfile": "ruby", + "Dockerfile": "dockerfile", + "Makefile": "make", + "mix.exs": "elixir", + } + for fname, expected in anchors.items(): + got = detect_language(fname) + assert got == expected, ( + f"detect_language({fname!r}) = {got!r}, expected {expected!r}" + ) + + def test_extension_map_has_no_duplicates(self): + """All extension values are canonical language names.""" + # All values must be lowercase, non-empty strings. + for ext, lang in EXTENSION_MAP.items(): + assert ext.startswith("."), f"extension {ext!r} must start with '.'" + assert ext == ext.lower(), f"extension {ext!r} must be lowercase" + assert lang and lang == lang.lower(), ( + f"language for {ext!r} must be non-empty lowercase, got {lang!r}" + ) + + +# ─── 2. detect_language edge cases ───────────────────────────── + + +class TestDetectLanguage: + """Edge cases for ``detect_language``.""" + + def test_empty_path_returns_none(self): + assert detect_language("") is None + + def test_none_like_path_returns_none(self): + # Path with no extension and no recognizable basename. + assert detect_language("README") is None + + def test_extension_case_insensitive(self): + assert detect_language("FOO.PY") == "python" + assert detect_language("Main.GO") == "go" + assert detect_language("App.RB") == "ruby" + + def test_dotted_filename_no_extension(self): + # Files like `.bashrc` or `.gitignore` have no real extension. + # We don't crash; we just return None or a detected language. + result = detect_language(".bashrc") + # ``.bashrc`` ends with ``bashrc`` which is not a known extension. + # Behavior: return None (graceful). + assert result is None + + def test_path_components_only_basename_used(self): + # Directories named ``.py`` shouldn't influence detection. + assert detect_language("/foo/.py/bar.go") == "go" + assert detect_language("/.rb/x.go") == "go" + + def test_dockerfile_with_prefix(self): + # Files like ``nginx.Dockerfile`` should still detect as dockerfile. + assert detect_language("nginx.Dockerfile") == "dockerfile" + assert detect_language("api.Containerfile") == "dockerfile" + + def test_makefile_lowercase(self): + # Basename matching is case-insensitive. + assert detect_language("makefile") == "make" + assert detect_language("MAKEFILE") == "make" + + def test_special_basenames(self): + assert detect_language("Rakefile") == "ruby" + assert detect_language("Gemfile") == "ruby" + assert detect_language("Capfile") == "ruby" + assert detect_language("Vagrantfile") == "ruby" + + def test_tsx_vs_tsx_extension(self): + assert detect_language("Component.tsx") == "tsx" + assert detect_language("module.ts") == "typescript" + + def test_header_files(self): + assert detect_language("foo.h") == "c" + assert detect_language("foo.hpp") == "cpp" + assert detect_language("foo.hxx") == "cpp" + + def test_shebang_python(self, tmp_path): + f = tmp_path / "script" + f.write_text("#!/usr/bin/env python3\nprint('hi')\n") + assert detect_language(str(f)) == "python" + + def test_shebang_bash(self, tmp_path): + f = tmp_path / "script" + f.write_text("#!/bin/bash\necho hi\n") + assert detect_language(str(f)) == "bash" + + def test_shebang_ruby(self, tmp_path): + f = tmp_path / "script" + f.write_text("#!/usr/bin/env ruby\nputs 'hi'\n") + assert detect_language(str(f)) == "ruby" + + def test_shebang_node(self, tmp_path): + f = tmp_path / "script" + f.write_text("#!/usr/bin/env node\nconsole.log('hi')\n") + assert detect_language(str(f)) == "javascript" + + def test_shebang_with_version_suffix(self, tmp_path): + # ``python3.11`` should still detect as ``python``. + f = tmp_path / "script" + f.write_text("#!/usr/bin/python3.11\nprint('hi')\n") + assert detect_language(str(f)) == "python" + + def test_no_shebang_extensionless_file(self, tmp_path): + f = tmp_path / "binary" + f.write_bytes(b"\x7fELF\x02\x01\x01\x00") + assert detect_language(str(f)) is None + + def test_missing_file_falls_back_to_extension(self, tmp_path): + # Path doesn't exist → extension check still works. + assert detect_language(str(tmp_path / "ghost.py")) == "python" + assert detect_language(str(tmp_path / "ghost.go")) == "go" + + def test_sql_yaml_toml_json(self): + assert detect_language("schema.sql") == "sql" + assert detect_language("config.yaml") == "yaml" + assert detect_language("config.yml") == "yaml" + assert detect_language("pyproject.toml") == "toml" + assert detect_language("package.json") == "json" + assert detect_language("tsconfig.jsonc") == "json" + + def test_terraform_hcl(self): + assert detect_language("main.tf") == "hcl" + assert detect_language("vars.tfvars") == "hcl" + assert detect_language("any.hcl") == "hcl" + + def test_scala_sbt(self): + assert detect_language("build.sbt") == "scala" + + def test_php_blade_not_in_fallback(self): + assert detect_language("view.php") == "php" + + +# ─── 3. load_grammar contract ────────────────────────────────── + + +class TestLoadGrammar: + """``load_grammar`` is the heart of issue #18.""" + + def test_returns_none_for_empty(self): + assert load_grammar("") is None + assert load_grammar(" ") is None + + def test_returns_none_for_unknown_language(self): + # Klingon isn't a real tree-sitter language. + assert load_grammar("klingon") is None + + def test_returns_none_for_none_input(self): + # Defensive: callers may pass None. + assert load_grammar(None) is None # type: ignore[arg-type] + + def test_loads_python_grammar(self): + # tree-sitter-python is installed (it's in the dev dependencies). + lang = load_grammar("python") + if lang is None: + pytest.skip("tree-sitter-python not installed in this env") + # The Language object should expose a name attribute or be truthy. + assert lang is not None + # ``Language`` objects from tree-sitter 0.22+ expose ``name``. + name = getattr(lang, "name", None) + assert name in (None, "python") # tolerate either form + + def test_normalizes_language_aliases(self): + # Hyphens and spaces normalize to underscores. + # ``c-sharp`` → ``csharp``, ``tree-sitter-c-sharp`` is the package. + # We only check that normalization doesn't crash and returns None + # (since c-sharp grammar isn't installed in CI). + assert load_grammar("CSharp") is None # case-insensitive normalization + assert load_grammar("csharp") is None + + def test_never_auto_installs_by_default(self, monkeypatch): + """Without the env var, ``load_grammar`` MUST NOT call pip.""" + # Ensure the env var is unset. + monkeypatch.delenv(AUTO_INSTALL_ENV, raising=False) + + calls = [] + + def fake_pip_install(package): + calls.append(package) + return False + + # Use monkeypatch on the module-level function. + import universal_grammar_loader as ugl + monkeypatch.setattr(ugl, "_pip_install", fake_pip_install) + + # Request a language whose grammar isn't installed. + result = load_grammar("zig") + assert result is None + assert calls == [], ( + "load_grammar() must NOT auto-install without " + f"{AUTO_INSTALL_ENV}=1, but called pip with: {calls}" + ) + + def test_auto_installs_when_env_var_set(self, monkeypatch): + """With the env var, ``load_grammar`` SHOULD call pip for missing grammars.""" + monkeypatch.setenv(AUTO_INSTALL_ENV, "1") + + calls = [] + + def fake_pip_install(package): + calls.append(package) + # Pretend install failed so we don't actually try to import. + return False + + import universal_grammar_loader as ugl + monkeypatch.setattr(ugl, "_pip_install", fake_pip_install) + + result = load_grammar("zig") + assert result is None # install failed → None + assert calls == ["tree-sitter-zig"], ( + f"expected pip install of tree-sitter-zig, got: {calls}" + ) + + def test_auto_install_truthy_values(self, monkeypatch): + """Various truthy env-var values enable auto-install.""" + import universal_grammar_loader as ugl + + for val in ("1", "true", "TRUE", "yes", "YES"): + monkeypatch.setenv(AUTO_INSTALL_ENV, val) + assert ugl._auto_install_enabled() is True, ( + f"{AUTO_INSTALL_ENV}={val!r} should enable auto-install" + ) + + def test_auto_install_falsy_values(self, monkeypatch): + """Empty / unset / unknown values disable auto-install.""" + import universal_grammar_loader as ugl + + for val in ("", "0", "no", "false", "random"): + monkeypatch.setenv(AUTO_INSTALL_ENV, val) + assert ugl._auto_install_enabled() is False, ( + f"{AUTO_INSTALL_ENV}={val!r} should disable auto-install" + ) + + def test_auto_install_disabled_when_unset(self, monkeypatch): + monkeypatch.delenv(AUTO_INSTALL_ENV, raising=False) + import universal_grammar_loader as ugl + assert ugl._auto_install_enabled() is False + + def test_returns_language_after_successful_install(self, monkeypatch): + """When pip install succeeds and the import works, return Language.""" + # Use python — the package is already installed so we short-circuit + # through the import path. We simulate a "fresh install" by: + # 1. Forcing the env var on. + # 2. Stubbing ``_try_import`` to first return None, then return + # the real tree_sitter_python module. + # 3. Stubbing ``_pip_install`` to return True. + monkeypatch.setenv(AUTO_INSTALL_ENV, "1") + + import universal_grammar_loader as ugl + + original_import = ugl._try_import + call_count = {"n": 0} + + def fake_import(language): + call_count["n"] += 1 + # First call (before install) returns None. + if call_count["n"] == 1: + return None + # Second call (after install) returns the real module. + return original_import(language) + + def fake_pip_install(package): + return True + + monkeypatch.setattr(ugl, "_try_import", fake_import) + monkeypatch.setattr(ugl, "_pip_install", fake_pip_install) + + lang = load_grammar("python") + assert lang is not None, "expected Language after successful auto-install" + assert call_count["n"] == 2, "import must be attempted twice (pre+post install)" + + def test_no_crash_on_broken_module(self, monkeypatch): + """A grammar module that raises during import shouldn't crash.""" + import universal_grammar_loader as ugl + + def broken_import(language): + raise RuntimeError("simulated broken wheel") + + monkeypatch.setattr(ugl, "_try_import", broken_import) + # Should return None, not raise. + assert load_grammar("python") is None + + def test_no_crash_on_broken_language_pointer(self, monkeypatch): + """A grammar module whose language() raises shouldn't crash.""" + import universal_grammar_loader as ugl + import types as _types + + # Build a fake module with a broken language() function. + fake_module = _types.SimpleNamespace() + def broken_language(): + raise RuntimeError("grammar corrupted") + fake_module.language = broken_language + + monkeypatch.setattr(ugl, "_try_import", lambda lang: fake_module) + assert load_grammar("python") is None + + +# ─── 4. GrammarLoader integration ────────────────────────────── + + +class TestGrammarLoaderIntegration: + """The legacy ``GrammarLoader`` class should delegate to the universal loader.""" + + def test_supported_languages_includes_universal_set(self): + langs = GrammarLoader.supported_languages() + # The universal loader covers far more than the original 7. + assert len(langs) >= 50, ( + f"GrammarLoader.supported_languages() should reflect universal " + f"loader (≥50 languages), got {len(langs)}" + ) + # Original 7 must still be present. + for lang in ("python", "javascript", "typescript", "tsx", + "rust", "html", "css"): + assert lang in langs, f"missing legacy language: {lang}" + + def test_available_languages_returns_list(self): + langs = GrammarLoader.available_languages() + assert isinstance(langs, list) + # tree-sitter-python is installed in dev env. + assert "python" in langs + + def test_get_language_python(self): + # Clear singleton cache to ensure fresh load. + GrammarLoader._instance = None + loader = GrammarLoader() + lang = loader.get_language("python") + if lang is None: + pytest.skip("tree-sitter-python not installed in this env") + assert lang is not None + + def test_get_language_unknown_returns_none(self): + GrammarLoader._instance = None + loader = GrammarLoader() + assert loader.get_language("klingon") is None + + def test_get_parser_python(self): + GrammarLoader._instance = None + loader = GrammarLoader() + parser = loader.get_parser("python") + if parser is None: + pytest.skip("tree-sitter-python not installed in this env") + # Should be able to parse a simple Python snippet. + tree = parser.parse(b"def f():\n return 1\n") + assert tree.root_node.type == "module" + + +# ─── 5. scan command integration ─────────────────────────────── + + +class TestScanIntegration: + """The ``scan`` command should use ``detect_language`` for unknown extensions.""" + + def test_unknown_extensions_bucketed_by_language(self, tmp_path): + """Files with unknown extensions are bucketed under detected language.""" + from commands.scan import discover_files + + # Create files whose extensions ARE in the universal loader but + # NOT in scan's hardcoded dispatch chain. + for fname in ("schema.sql", "config.yaml", "pyproject.toml", "main.tf"): + (tmp_path / fname).write_text("# placeholder\n") + + config = {"ignore_dirs": [], "ignore_exts": []} + files = discover_files(str(tmp_path), config) + + # Each detected file should land in a bucket named after its language. + assert "sql" in files and len(files["sql"]) == 1 + assert "yaml" in files and len(files["yaml"]) == 1 + assert "toml" in files and len(files["toml"]) == 1 + assert "hcl" in files and len(files["hcl"]) == 1 + + def test_known_extensions_still_use_hardcoded_buckets(self, tmp_path): + """Files with known extensions keep using the legacy parser buckets.""" + from commands.scan import discover_files + + (tmp_path / "main.go").write_text("package main\n") + (tmp_path / "app.rb").write_text("puts 'hi'\n") + (tmp_path / "Dockerfile").write_text("FROM alpine\n") + config = {"ignore_dirs": [], "ignore_exts": []} + files = discover_files(str(tmp_path), config) + + # .go and .rb keep their legacy buckets — universal loader is + # a *fallback*, not a replacement for the curated dispatch. + assert len(files["go"]) == 1 + assert len(files["ruby"]) == 1 + # Dockerfile is dispatched to the shell bucket by the hardcoded + # chain (existing behavior, preserved for backward compat). + assert len(files["shell"]) == 1 + + +# ─── 6. Module-level contracts ───────────────────────────────── + + +class TestModuleContracts: + """Sanity checks on the public surface of the module.""" + + def test_public_api_exports(self): + import universal_grammar_loader as ugl + for name in ("detect_language", "load_grammar", + "available_languages", "supported_languages", + "supported_extensions_count", + "EXTENSION_MAP", "BASENAME_MAP", "SHEBANG_MAP", + "AUTO_INSTALL_ENV"): + assert hasattr(ugl, name), f"missing public symbol: {name}" + + def test_auto_install_env_constant(self): + assert AUTO_INSTALL_ENV == "CODELENS_AUTO_INSTALL_GRAMMARS" + + def test_shebang_map_covers_common_interpreters(self): + for interp in ("python", "python3", "ruby", "bash", "node"): + assert interp in SHEBANG_MAP, ( + f"shebang mapping missing for {interp!r}" + ) + + def test_no_two_extensions_map_to_different_canonical_names_for_same_lang(self): + """Sanity: extensions grouped under the same canonical language name.""" + # All extensions mapped to ``python`` should produce ``python``. + python_exts = [e for e, l in EXTENSION_MAP.items() if l == "python"] + assert ".py" in python_exts + for ext in python_exts: + assert detect_language("file" + ext) == "python" + + def test_supported_languages_returns_sorted_tuple(self): + langs = supported_languages() + assert isinstance(langs, tuple) + assert langs == tuple(sorted(langs))