diff --git a/graphify/detect.py b/graphify/detect.py index 92b773f7b..105d8f100 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -32,6 +32,9 @@ class FileType(str, Enum): PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} +# Notebooks are converted to markdown sidecars before indexing — do NOT add .ipynb +# to CODE_EXTENSIONS or DOC_EXTENSIONS. +NOTEBOOK_EXTENSIONS = {'.ipynb'} VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} CORPUS_WARN_THRESHOLD = 50_000 # words - below this, warn "you may not need a graph" @@ -413,6 +416,8 @@ def classify_file(path: Path) -> FileType | None: return FileType.DOCUMENT if ext in OFFICE_EXTENSIONS: return FileType.DOCUMENT + if ext in NOTEBOOK_EXTENSIONS: + return FileType.DOCUMENT if ext in GOOGLE_WORKSPACE_EXTENSIONS: return FileType.DOCUMENT if ext in VIDEO_EXTENSIONS: @@ -642,6 +647,68 @@ def convert_office_file(path: Path, out_dir: Path) -> Path | None: return out_path +def ipynb_to_markdown(path: Path) -> str: + """Convert a Jupyter notebook to markdown, stripping outputs. + + Uses the notebook's kernel language from metadata for fenced code blocks, + falling back to ``code`` when the metadata is absent. + """ + if not _file_within_size_cap(path): + return "" + try: + nb = json.loads(path.read_text(encoding="utf-8", errors="ignore")) + # Resolve the kernel language from notebook metadata so fenced code + # blocks use the correct language identifier (e.g. ```python) rather + # than the generic ```code fallback. + meta = nb.get("metadata", {}) + lang = ( + meta.get("language_info", {}).get("name") + or meta.get("kernelspec", {}).get("language") + or "code" + ) + lines = [] + for cell in nb.get("cells", []): + ct = cell.get("cell_type") + raw_src = cell.get("source", []) + src = raw_src if isinstance(raw_src, str) else "".join(raw_src) + if not src.strip(): + continue + if ct == "markdown": + lines.append(src) + elif ct == "code": + lines.append(f"```{lang}\n{src}\n```") + return "\n\n".join(lines) + except Exception: + return "" + + +def convert_notebook_file(path: Path, out_dir: Path) -> Path | None: + """Convert a .ipynb to a markdown sidecar in out_dir. + + Mirrors convert_office_file() sidecar naming. Unlike office files, an + existing sidecar is refreshed when cell sources change but left untouched + when only outputs/metadata changed — notebook re-runs must not re-extract. + """ + if path.suffix.lower() not in NOTEBOOK_EXTENSIONS: + return None + + text = ipynb_to_markdown(path) + if not text.strip(): + return None + + out_dir.mkdir(parents=True, exist_ok=True) + import hashlib + import unicodedata + normalized_path = unicodedata.normalize("NFC", str(path.resolve())) + name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] + out_path = out_dir / f"{path.stem}_{name_hash}.md" + payload = f"\n\n{text}" + if out_path.exists() and out_path.read_text(encoding="utf-8") == payload: + return out_path + out_path.write_text(payload, encoding="utf-8") + return out_path + + def count_words(path: Path) -> int: try: ext = path.suffix.lower() @@ -1131,6 +1198,16 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: # Conversion failed (library not installed) - skip with note skipped_sensitive.append(str(p) + " [office conversion failed - pip install graphifyy[office]]") continue + if p.suffix.lower() in NOTEBOOK_EXTENSIONS: + md_path = convert_notebook_file(p, converted_dir) + if md_path: + if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): + continue + files[ftype].append(str(md_path)) + total_words += count_words(md_path) + else: + skipped_sensitive.append(str(p) + " [notebook conversion failed]") + continue files[ftype].append(str(p)) if ftype != FileType.VIDEO: total_words += count_words(p) diff --git a/tests/test_detect.py b/tests/test_detect.py index 76282f295..e183dabad 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1515,3 +1515,184 @@ def test_convert_office_file_does_not_rewrite_existing_sidecar(tmp_path, monkeyp second = detect_mod.convert_office_file(src, out_dir) assert second == first assert second.stat().st_mtime_ns == mtime_before + + +def _minimal_ipynb(cells): + import json + return json.dumps({"cells": cells, "nbformat": 4, "nbformat_minor": 5}) + + +def test_classify_ipynb(): + assert classify_file(Path("analysis.ipynb")) == FileType.DOCUMENT + + +def test_ipynb_to_markdown_mixed_cells(tmp_path): + nb_path = tmp_path / "nb.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "markdown", "source": "# Title\n\nIntro text."}, + { + "cell_type": "code", + "source": "import pandas as pd\nprint('hi')", + "outputs": [{"output_type": "stream", "text": "hi\n"}], + }, + {"cell_type": "markdown", "source": "## Section"}, + ]), + encoding="utf-8", + ) + md = detect_mod.ipynb_to_markdown(nb_path) + assert "# Title" in md + assert "```python\nimport pandas as pd" in md + assert "hi\n" not in md # outputs stripped + assert "## Section" in md + assert md.index("# Title") < md.index("```python") < md.index("## Section") + + +def test_ipynb_to_markdown_empty_notebook(tmp_path): + nb_path = tmp_path / "empty.ipynb" + nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") + assert detect_mod.ipynb_to_markdown(nb_path) == "" + + +def test_ipynb_to_markdown_malformed_json(tmp_path): + nb_path = tmp_path / "bad.ipynb" + nb_path.write_text("{not valid json", encoding="utf-8") + assert detect_mod.ipynb_to_markdown(nb_path) == "" + + +def test_detect_converts_notebook_to_sidecar(tmp_path): + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "markdown", "source": "# Analysis"}, + {"cell_type": "code", "source": "x = 1"}, + ]), + encoding="utf-8", + ) + result = detect(tmp_path) + assert len(result["files"]["document"]) == 1 + sidecar = Path(result["files"]["document"][0]) + assert sidecar.suffix == ".md" + assert sidecar.exists() + text = sidecar.read_text(encoding="utf-8") + assert "converted from analysis.ipynb" in text + assert "# Analysis" in text + assert "x = 1" in text + assert result["total_words"] > 0 + + +def test_convert_notebook_file_empty_notebook_no_sidecar(tmp_path): + nb_path = tmp_path / "empty.ipynb" + nb_path.write_text(_minimal_ipynb([]), encoding="utf-8") + out_dir = tmp_path / "converted" + assert detect_mod.convert_notebook_file(nb_path, out_dir) is None + assert not list(out_dir.glob("*.md")) + + +def test_convert_notebook_file_output_change_preserves_sidecar_mtime(tmp_path): + """Re-running a notebook updates outputs in the .ipynb but not the sidecar.""" + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([ + {"cell_type": "code", "source": "print(1)", "outputs": []}, + ]), + encoding="utf-8", + ) + out_dir = tmp_path / "converted" + sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) + assert sidecar is not None + mtime_before = sidecar.stat().st_mtime_ns + + nb_path.write_text( + _minimal_ipynb([ + { + "cell_type": "code", + "source": "print(1)", + "outputs": [{"output_type": "stream", "text": "1\n"}], + "execution_count": 1, + }, + ]), + encoding="utf-8", + ) + again = detect_mod.convert_notebook_file(nb_path, out_dir) + assert again == sidecar + assert again.stat().st_mtime_ns == mtime_before + + +def test_convert_notebook_file_source_change_rewrites_sidecar(tmp_path): + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 1", "outputs": []}]), + encoding="utf-8", + ) + out_dir = tmp_path / "converted" + sidecar = detect_mod.convert_notebook_file(nb_path, out_dir) + mtime_before = sidecar.stat().st_mtime_ns + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "x = 2", "outputs": []}]), + encoding="utf-8", + ) + updated = detect_mod.convert_notebook_file(nb_path, out_dir) + assert updated == sidecar + assert "x = 2" in updated.read_text(encoding="utf-8") + assert updated.stat().st_mtime_ns >= mtime_before + + +def test_detect_refreshes_notebook_sidecar_on_source_change(tmp_path): + """Cell source edits must update the sidecar so a later extract sees new content.""" + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v1"}]), + encoding="utf-8", + ) + detect(tmp_path) + converted_dir = tmp_path / "graphify-out" / "converted" + sidecar = next(converted_dir.glob("analysis_*.md")) + + nb_path.write_text( + _minimal_ipynb([{"cell_type": "markdown", "source": "v2 updated"}]), + encoding="utf-8", + ) + detect(tmp_path) + assert "v2 updated" in sidecar.read_text(encoding="utf-8") + + +def test_detect_incremental_ignores_notebook_output_only_changes(tmp_path): + import json + + nb_path = tmp_path / "analysis.ipynb" + nb_path.write_text( + _minimal_ipynb([{"cell_type": "code", "source": "print(1)", "outputs": []}]), + encoding="utf-8", + ) + first = detect(tmp_path) + sidecar = Path(first["files"]["document"][0]) + mtime_before = sidecar.stat().st_mtime_ns + manifest_path = tmp_path / "graphify-out" / "manifest.json" + Path(manifest_path).write_text( + json.dumps({ + str(sidecar): { + "mtime": sidecar.stat().st_mtime, + "ast_hash": "a" * 32, + "semantic_hash": "b" * 32, + } + }), + encoding="utf-8", + ) + + nb_path.write_text( + _minimal_ipynb([ + { + "cell_type": "code", + "source": "print(1)", + "outputs": [{"output_type": "stream", "text": "1\n"}], + "execution_count": 1, + }, + ]), + encoding="utf-8", + ) + inc = detect_incremental(tmp_path, manifest_path=str(manifest_path)) + assert sidecar.stat().st_mtime_ns == mtime_before + assert not inc["new_files"]["document"] + assert str(sidecar) in inc["unchanged_files"]["document"]