diff --git a/scripts/persistent_registry.py b/scripts/persistent_registry.py index 2266665c..d9357313 100644 --- a/scripts/persistent_registry.py +++ b/scripts/persistent_registry.py @@ -680,6 +680,160 @@ def load_backend_registry(self) -> Optional[Dict[str, Any]]: """Load the backend registry from the cache.""" return self.get_cached_result("__backend_registry__", ["__all__"]) + # ─── Scan Result Storage ──────────────────────────── + + # Directories that are never part of the scanned file set, used by + # _compute_scan_file_set_hash when walking the workspace. Kept short + # and explicit so the hash reflects only user source code. + _SCAN_HASH_IGNORED_DIRS = { + ".codelens", ".git", "node_modules", ".venv", "venv", + "__pycache__", ".pytest_cache", ".idea", ".vscode", + } + + def _compute_scan_file_set_hash(self, workspace: str) -> str: + """Compute a stable SHA-256 hash for the set of source files in workspace. + + Walks the workspace with ``os.walk``, skips well-known non-source + directories (``.codelens``, ``.git``, ``node_modules``, etc.), sorts + the resulting relative paths, and hashes the joined string. The hash + is stable across runs as long as the file set is unchanged — files + can be edited without invalidating the cache key, which matches the + design intent of ``analysis_cache`` (per the module docstring at + L7-15: cache keyed by ``(command, file_set_hash)``). + + Args: + workspace: Absolute path to the workspace root + + Returns: + 64-char hex SHA-256 digest, or empty string if workspace + does not exist or contains no source files. + """ + if not workspace or not os.path.isdir(workspace): + return "" + + rel_paths: List[str] = [] + for root, dirs, filenames in os.walk(workspace): + # Prune ignored directories in-place so os.walk does not descend + dirs[:] = [d for d in dirs if d not in self._SCAN_HASH_IGNORED_DIRS] + for filename in filenames: + abs_path = os.path.join(root, filename) + rel_path = os.path.relpath(abs_path, workspace) + # Normalize to forward slashes for cross-platform stability + rel_paths.append(rel_path.replace(os.sep, "/")) + + if not rel_paths: + return "" + + rel_paths.sort() + joined = "\n".join(rel_paths) + return hashlib.sha256(joined.encode("utf-8")).hexdigest() + + def store_scan_result(self, result: Dict[str, Any]) -> None: + """Persist a normalized subset of a scan result to ``analysis_cache``. + + Closes the P0 silent bug from issue #31: ``codelens.py`` calls + ``pr.store_scan_result(result)`` after every successful scan, but + the method did not exist, raising ``AttributeError`` that was + swallowed by a bare ``except Exception`` — so the + ``analysis_cache`` table was never populated, the + ``result["sqlite_persisted"] = True`` flag was never set, and the + success message was never printed. + + This method persists a small, normalized subset of the scan result + (counts + frameworks + timestamp), keyed by ``command="scan"`` and + a ``file_set_hash`` derived from the workspace's source file paths. + Idempotent: re-scanning the same file set with the same result + subset does not insert a duplicate row. + + Args: + result: The full scan result dict returned by + ``commands.scan.cmd_scan``. Expected to contain the + ``workspace``, ``files_scanned``, ``frontend``, + ``backend``, and ``frameworks`` keys; missing keys are + tolerated and contribute zero/empty values to the subset. + """ + workspace = result.get("workspace") or self.workspace + + files_scanned = result.get("files_scanned", {}) or {} + frontend = result.get("frontend", {}) or {} + backend = result.get("backend", {}) or {} + frameworks = result.get("frameworks", []) or [] + + # total_symbols mirrors the calc used by migrate_from_json + # (frontend_classes + frontend_ids + backend_nodes) and gives + # a single comparable scalar for trend tracking. + total_symbols = ( + int(frontend.get("classes", 0) or 0) + + int(frontend.get("ids", 0) or 0) + + int(backend.get("nodes", 0) or 0) + ) + + subset = { + "files_scanned": files_scanned, + "frontend_counts": { + "classes": int(frontend.get("classes", 0) or 0), + "ids": int(frontend.get("ids", 0) or 0), + }, + "backend_counts": { + "nodes": int(backend.get("nodes", 0) or 0), + "edges": int(backend.get("edges", 0) or 0), + }, + "frameworks": frameworks, + "scan_timestamp": time.time(), + "total_symbols": total_symbols, + } + + file_set_hash = self._compute_scan_file_set_hash(workspace) + if not file_set_hash: + logger.warning( + "store_scan_result: empty file_set_hash (workspace=%s); " + "skipping analysis_cache insert", workspace + ) + return + + result_json = json.dumps(subset, ensure_ascii=False, default=str, sort_keys=True) + result_hash = hashlib.sha256(result_json.encode("utf-8")).hexdigest() + now = time.time() + + conn = self._connect() + # Idempotent: if the same (command, file_set_hash, result_hash) + # triple already exists, do nothing. This mirrors the "INSERT OR + # IGNORE" intent — re-scanning the same file set with the same + # outcome must not bloat the cache. + existing = conn.execute( + """SELECT 1 FROM analysis_cache + WHERE command = ? AND file_set_hash = ? AND result_hash = ? + LIMIT 1 + """, + ("scan", file_set_hash, result_hash), + ).fetchone() + if existing: + logger.debug( + "store_scan_result: cache hit (command=scan, " + "file_set_hash=%s, result_hash=%s) — skipping insert", + file_set_hash[:12], result_hash[:12], + ) + return + + try: + conn.execute( + """INSERT INTO analysis_cache + (command, file_set_hash, result_hash, result_json, timestamp) + VALUES (?, ?, ?, ?, ?) + """, + ("scan", file_set_hash, result_hash, result_json, now), + ) + conn.commit() + logger.info( + "store_scan_result: persisted scan result to analysis_cache " + "(file_set_hash=%s, total_symbols=%d)", + file_set_hash[:12], total_symbols, + ) + except sqlite3.Error as e: + logger.warning( + "store_scan_result: failed to insert into analysis_cache: %s", e + ) + # ─── Migration ────────────────────────────────────────── def migrate_from_json(self) -> Dict[str, Any]: diff --git a/tests/test_persistent_registry.py b/tests/test_persistent_registry.py index 9bc91de6..cf1412af 100644 --- a/tests/test_persistent_registry.py +++ b/tests/test_persistent_registry.py @@ -473,3 +473,189 @@ def test_migrate_already_exists(self, workspace): result = cmd_migrate(workspace) assert result['status'] == 'ok' assert 'already exists' in result.get('message', '') + + +class TestStoreScanResult: + """Test PersistentRegistry.store_scan_result — closes issue #31. + + The P0 bug: codelens.py calls pr.store_scan_result(result) after every + successful scan, but the method did not exist. The resulting + AttributeError was swallowed by a bare except Exception, so the + analysis_cache table was never populated and the sqlite_persisted + flag was never set. These tests verify the fix. + """ + + def _make_scan_result(self, workspace, **overrides): + """Build a minimal scan result dict matching cmd_scan's shape.""" + result = { + "status": "ok", + "workspace": workspace, + "files_scanned": { + "python": 2, + "css": 1, + "html": 1, + }, + "frontend": {"classes": 3, "ids": 1}, + "backend": {"nodes": 4, "edges": 2}, + "frameworks": ["react"], + } + result.update(overrides) + return result + + def test_store_scan_result_persists_to_analysis_cache(self, populated_workspace): + """store_scan_result writes exactly one row to analysis_cache.""" + from persistent_registry import PersistentRegistry + import sqlite3 + + reg = PersistentRegistry(populated_workspace) + reg._connect() + + result = self._make_scan_result(populated_workspace) + reg.store_scan_result(result) + + db_path = reg.db_path + reg.close() + + # Query analysis_cache directly to verify the row was written + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + rows = conn.execute( + "SELECT command, file_set_hash, result_hash, result_json, timestamp " + "FROM analysis_cache WHERE command = 'scan'" + ).fetchall() + conn.close() + + assert len(rows) == 1, f"expected 1 scan row, got {len(rows)}" + row = rows[0] + assert row["command"] == "scan" + assert row["file_set_hash"], "file_set_hash must be non-empty" + assert len(row["file_set_hash"]) == 64, "file_set_hash must be SHA-256 hex" + assert row["result_hash"], "result_hash must be non-empty" + assert row["timestamp"] > 0, "timestamp must be set" + # The stored JSON must contain the normalized subset fields + import json as _json + stored = _json.loads(row["result_json"]) + assert "files_scanned" in stored + assert "frontend_counts" in stored + assert "backend_counts" in stored + assert "frameworks" in stored + assert "scan_timestamp" in stored + assert "total_symbols" in stored + # total_symbols = fe.classes + fe.ids + be.nodes = 3 + 1 + 4 = 8 + assert stored["total_symbols"] == 8 + + def test_store_scan_result_is_idempotent(self, populated_workspace): + """Calling store_scan_result twice with the same result inserts only one row.""" + from persistent_registry import PersistentRegistry + import sqlite3 + + reg = PersistentRegistry(populated_workspace) + reg._connect() + + result = self._make_scan_result(populated_workspace) + # Note: scan_timestamp is computed inside store_scan_result via + # time.time(), so the second call will produce a DIFFERENT subset + # unless we freeze the timestamp. To test idempotency, we patch + # time.time inside the module to return a fixed value. + import persistent_registry as pr_module + + original_time = pr_module.time.time + pr_module.time.time = lambda: 1700000000.0 + try: + reg.store_scan_result(result) + reg.store_scan_result(result) + finally: + pr_module.time.time = original_time + + db_path = reg.db_path + reg.close() + + conn = sqlite3.connect(db_path) + count = conn.execute( + "SELECT COUNT(*) FROM analysis_cache WHERE command = 'scan'" + ).fetchone()[0] + conn.close() + + assert count == 1, ( + f"expected 1 row after duplicate store_scan_result, got {count}" + ) + + def test_store_scan_result_different_file_sets(self, workspace): + """Two different file sets produce two rows with different file_set_hash.""" + from persistent_registry import PersistentRegistry + import sqlite3 + + reg = PersistentRegistry(workspace) + reg._connect() + + # File set A: two python files + with open(os.path.join(workspace, 'a.py'), 'w') as f: + f.write('x = 1\n') + with open(os.path.join(workspace, 'b.py'), 'w') as f: + f.write('y = 2\n') + result_a = self._make_scan_result(workspace, files_scanned={"python": 2}) + reg.store_scan_result(result_a) + + # File set B: add a third file -> different file set -> different hash + with open(os.path.join(workspace, 'c.py'), 'w') as f: + f.write('z = 3\n') + result_b = self._make_scan_result(workspace, files_scanned={"python": 3}) + # Force a different result_hash by changing backend counts + result_b["backend"] = {"nodes": 99, "edges": 0} + reg.store_scan_result(result_b) + + db_path = reg.db_path + reg.close() + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + rows = conn.execute( + "SELECT file_set_hash, result_hash FROM analysis_cache " + "WHERE command = 'scan' ORDER BY file_set_hash" + ).fetchall() + conn.close() + + assert len(rows) == 2, f"expected 2 rows for 2 file sets, got {len(rows)}" + assert rows[0]["file_set_hash"] != rows[1]["file_set_hash"], ( + "file_set_hash must differ for different file sets" + ) + + def test_scan_command_sets_sqlite_persisted_flag(self, populated_workspace): + """Running the actual scan command via cmd_scan sets sqlite_persisted=True. + + This is the end-to-end test that reproduces the original P0 bug: + before the fix, AttributeError was raised inside the post-scan + block and silently swallowed, so sqlite_persisted was never set. + """ + from commands.scan import cmd_scan + from persistent_registry import PersistentRegistry + + # Run the actual scan command (this exercises the real call site + # at codelens.py:977 indirectly — cmd_scan returns the result, + # and the post-scan block in codelens.py wraps it. We test the + # contract here by simulating the same logic.) + result = cmd_scan(populated_workspace, incremental=False) + + # Simulate the codelens.py post-scan SQLite block (L971-983) + from persistent_registry import is_sqlite_available + if is_sqlite_available(): + pr = PersistentRegistry(populated_workspace) + pr.store_scan_result(result) + result["sqlite_persisted"] = True + pr.close() + + assert result.get("sqlite_persisted") is True, ( + "sqlite_persisted flag must be True after successful scan + " + "store_scan_result (was False before issue #31 fix)" + ) + + # Verify the row actually landed in analysis_cache + import sqlite3 + conn = sqlite3.connect(os.path.join( + populated_workspace, ".codelens", "codelens.db" + )) + count = conn.execute( + "SELECT COUNT(*) FROM analysis_cache WHERE command = 'scan'" + ).fetchone()[0] + conn.close() + assert count >= 1, "analysis_cache must have at least 1 scan row"