From 6529aa838df788417459151c3004c41ca622b9ca Mon Sep 17 00:00:00 2001 From: Wolfvin Date: Fri, 3 Jul 2026 14:53:21 +0000 Subject: [PATCH] fix(semantic-query): read symbols from graph_nodes, not empty symbols table (closes #188) --- scripts/semantic_search_engine.py | 81 +++++++++++++++++++++++----- tests/test_semantic_search_engine.py | 70 +++++++++++++----------- 2 files changed, 108 insertions(+), 43 deletions(-) diff --git a/scripts/semantic_search_engine.py b/scripts/semantic_search_engine.py index 1d349959..bed326e8 100644 --- a/scripts/semantic_search_engine.py +++ b/scripts/semantic_search_engine.py @@ -13,10 +13,13 @@ or any ~80 MB embedding model. Fast to import, deterministic, no native deps. Good enough for the majority of agent "find the right file" queries. * **Reads from the existing SQLite registry.** Symbol text is built from - fields already populated by ``persistent_registry`` — ``name``, - ``signature``, ``kind``, ``file_path``, ``language`` — so the index is - always in sync with the last ``scan`` result. No separate index file to - keep consistent. + the ``graph_nodes`` table populated by ``scan`` via + :func:`graph_model.populate_graph_tables` — fields ``name``, + ``node_type``, ``file``, ``line``, ``extra_json`` — so the index is + always in sync with the last ``scan`` result. No separate index file + to keep consistent. (Issue #188: previously read from the ``symbols`` + table, which is never populated by the scan flow — see + :func:`_load_symbols_from_db` for details.) * **In-memory index, cached per (db_path, mtime).** Building the vocabulary is O(N_symbols) — for CodeLens's own 3000-node graph this is <100 ms. Cached so repeated ``semantic_query`` calls within a session are ~1 ms. @@ -379,8 +382,34 @@ def clear_cache() -> None: def _load_symbols_from_db(db_path: str) -> List[Dict[str, Any]]: """Load all symbols from the SQLite registry at ``db_path``. - Returns an empty list if the database doesn't exist, the ``symbols`` - table is missing, or SQLite is unavailable. Never raises — semantic + Reads from the ``graph_nodes`` table populated by ``scan`` via + :func:`graph_model.populate_graph_tables`. The legacy ``symbols`` + table declared by ``persistent_registry`` is never populated by the + scan flow (issue #188) — only the graph tables are — so reading from + ``symbols`` returned an empty result set on every real workspace. + + Column mapping (graph_nodes -> symbol dict consumed by this module): + + node_id -> id (string node id, e.g. "auth/jwt.py:42") + name -> name (symbol name) + file -> file_path (relative file path) + node_type -> kind (function|class|file|module|route|type|interface) + line -> line_start (1-indexed line number) + extra_json-> extra_json (preserves async/impl_for/status/...) + + Fields that ``graph_nodes`` does not carry (``signature``, + ``language``, ``line_end``, ``hash``) are defaulted to empty/None so + downstream consumers (:func:`_build_symbol_text`, :func:`semantic_query` + result construction) keep working unchanged. Signatures and language + are not part of the flat-registry node format produced by parsers + (see ``scripts/parsers/python_parser.py`` for a representative node), + so there is no signal loss in practice — the TF-IDF document is still + built from name + file_path + kind + extra_json, which is enough for + "find by meaning" queries. + + Returns an empty list if the database doesn't exist, the + ``graph_nodes`` table is missing (e.g. workspace initialized but + never scanned), or SQLite is unavailable. Never raises — semantic search is a non-breaking add-on and should degrade gracefully to "no results" rather than crash the host command. @@ -398,21 +427,47 @@ def _load_symbols_from_db(db_path: str) -> List[Dict[str, Any]]: conn = sqlite3.connect(db_path) try: conn.row_factory = sqlite3.Row - # Sanity-check that the symbols table exists. If the workspace - # has been initialized but never scanned, the db file exists - # but has no tables. + # Sanity-check that the graph_nodes table exists. If the + # workspace has been initialized but never scanned, the db + # file exists but has no graph tables. table = conn.execute( "SELECT name FROM sqlite_master " - "WHERE type='table' AND name='symbols'" + "WHERE type='table' AND name='graph_nodes'" ).fetchone() if table is None: return [] - rows = conn.execute("SELECT * FROM symbols").fetchall() - return [dict(r) for r in rows] + # Map graph_nodes columns to the symbol-dict shape expected + # by _build_symbol_text and semantic_query's result builder. + # Columns not present in graph_nodes are filled with defaults + # so the rest of the engine does not need to know about the + # source table. + rows = conn.execute( + """ + SELECT + node_id AS id, + name AS name, + node_type AS kind, + file AS file_path, + line AS line_start, + extra_json AS extra_json + FROM graph_nodes + """ + ).fetchall() + symbols: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + # Fill defaults for fields graph_nodes does not carry so + # _build_symbol_text / result construction keep working. + d.setdefault("line_end", None) + d.setdefault("language", "") + d.setdefault("signature", "") + d.setdefault("hash", "") + symbols.append(d) + return symbols finally: conn.close() except sqlite3.Error as e: - logger.debug(f"semantic_search_engine: failed to read symbols from {db_path}: {e}") + logger.debug(f"semantic_search_engine: failed to read graph_nodes from {db_path}: {e}") return [] diff --git a/tests/test_semantic_search_engine.py b/tests/test_semantic_search_engine.py index 08806963..b709bc7a 100644 --- a/tests/test_semantic_search_engine.py +++ b/tests/test_semantic_search_engine.py @@ -20,7 +20,7 @@ * IDF weighting causes rare, discriminative terms (e.g. ``jwt``) to rank higher than ubiquitous terms (e.g. ``function``). * The engine degrades gracefully — empty query, missing db, missing - ``symbols`` table, and ``top_k=0`` all return well-formed responses + ``graph_nodes`` table, and ``top_k=0`` all return well-formed responses rather than raising. * The cache invalidates on db mtime change, so a re-scan picks up new symbols. @@ -59,50 +59,58 @@ def _make_db(workspace: str, symbols): """Create a fake ``.codelens/codelens.db`` with the given symbol rows. - ``symbols`` is a list of dicts with keys: name, kind, file_path, - line_start, language, signature, extra_json (dict, will be - JSON-encoded). + Mirrors what ``scan`` actually produces: rows in the ``graph_nodes`` + table (issue #188). The engine reads from ``graph_nodes``, not the + legacy ``symbols`` table declared by ``persistent_registry`` (which + the scan flow never populates). + + ``symbols`` is a list of dicts with keys: name, kind (maps to + node_type), file_path (maps to file), line_start (maps to line), + language, signature, extra_json (dict, will be JSON-encoded). """ codelens_dir = os.path.join(workspace, ".codelens") os.makedirs(codelens_dir, exist_ok=True) db_path = os.path.join(codelens_dir, "codelens.db") conn = sqlite3.connect(db_path) try: + # Schema mirrors scripts/graph_model.py:_CREATE_GRAPH_NODES so the + # engine's SELECT AS mapping has real columns to read from. conn.execute( """ - CREATE TABLE symbols ( + CREATE TABLE graph_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT, + node_id TEXT NOT NULL UNIQUE, + node_type TEXT NOT NULL DEFAULT 'function', name TEXT NOT NULL, - kind TEXT NOT NULL DEFAULT 'function', - file_path TEXT, - line_start INTEGER, - line_end INTEGER, - language TEXT, - signature TEXT, - hash TEXT, + file TEXT, + line INTEGER, extra_json TEXT ) """ ) for i, s in enumerate(symbols, start=1): extra = s.get("extra_json") or {} + name = s["name"] + file_path = s.get("file_path", "") + line = s.get("line_start") or 0 + # Synthesize a node_id matching the flat-registry convention + # (file:line) — what populate_graph_tables would store. This + # keeps tests realistic and would let future graph-traversal + # integration reuse the same fixtures. + node_id = s.get("id") or f"{file_path}:{line}:{name}" conn.execute( """ - INSERT INTO symbols - (id, name, kind, file_path, line_start, line_end, - language, signature, hash, extra_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO graph_nodes + (id, node_id, node_type, name, file, line, extra_json) + VALUES (?, ?, ?, ?, ?, ?, ?) """, ( i, - s["name"], + node_id, s.get("kind", "function"), - s.get("file_path", ""), - s.get("line_start"), - s.get("line_end"), - s.get("language", ""), - s.get("signature", ""), - s.get("hash", ""), + name, + file_path, + line, json.dumps(extra) if extra else None, ), ) @@ -255,7 +263,7 @@ class TestSemanticQuery: def test_finds_auth_symbol_by_concept(self, auth_workspace): """The core issue #11 use case: a query token that doesn't appear in any symbol NAME still surfaces relevant symbols because file - paths, signatures, and kinds are all part of the TF-IDF document. + paths and kinds are part of the TF-IDF document. Here, the query ``"auth"`` doesn't appear in any symbol name (``verify_jwt_claims``, ``loginUser``, ``format_date``, etc.), but @@ -380,13 +388,14 @@ def test_cache_invalidates_on_mtime_change(self, workspace): idx1 = build_index(db_path) assert len(idx1.symbols) == 1 - # Force mtime change by sleeping then writing new symbols + # Force mtime change by sleeping then writing a new graph_nodes + # row (what a real re-scan would do). time.sleep(0.05) conn = sqlite3.connect(db_path) try: conn.execute( - "INSERT INTO symbols (name, kind, file_path, line_start, language) " - "VALUES ('bar', 'function', 'b.py', 2, 'python')" + "INSERT INTO graph_nodes (node_id, node_type, name, file, line) " + "VALUES ('b.py:2:bar', 'function', 'bar', 'b.py', 2)" ) conn.commit() finally: @@ -413,8 +422,9 @@ def test_missing_db_returns_empty_results(self, workspace): assert result["stats"]["total_symbols"] == 0 assert result["stats"]["returned"] == 0 - def test_db_without_symbols_table_returns_empty(self, workspace): - # Create the db file but no `symbols` table + def test_db_without_graph_nodes_table_returns_empty(self, workspace): + # Create the db file but no `graph_nodes` table — mirrors a + # workspace that has been `codelens init`-ed but never scanned. codelens_dir = os.path.join(workspace, ".codelens") os.makedirs(codelens_dir, exist_ok=True) db_path = os.path.join(codelens_dir, "codelens.db") @@ -427,7 +437,7 @@ def test_db_without_symbols_table_returns_empty(self, workspace): assert result["status"] == "ok" assert result["results"] == [] - def test_empty_symbols_table_returns_empty(self, workspace): + def test_empty_graph_nodes_table_returns_empty(self, workspace): _make_db(workspace, []) result = semantic_query(workspace, "anything", top_k=10) assert result["status"] == "ok"