From e5c07201f4b0e0a335afe55dfb3ef16d259ff414 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Fri, 10 Jul 2026 11:20:30 +0100 Subject: [PATCH] feat: script -> notebook converter (al_to_notebook) for narrative-docstring scripts (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib-only autoassistant/to_notebook.py (nbformat-v4 JSON direct; adapts PyAutoBuild autobuild/build_util.py py_to_notebook + add_notebook_quotes semantics, cited as reference — no PyAutoBuild or ipynb-py-convert dependency for assistant/project users); al_to_notebook skill + README registration + .claude/skills symlink; unit tests pin the cell split incl. the back-to-back docstring case the reference pipeline mis-renders. Verified on autolens_workspace start_here.py: 25/26 cells identical to the pipeline, the 26th is the pipeline's own artifact (docstring rendered as a code cell with literal '# %%' markers) which this converter handles correctly. Co-Authored-By: Claude Fable 5 --- .claude/skills/al_to_notebook.md | 1 + autoassistant/tests/test_to_notebook.py | 72 ++++++++++++++ autoassistant/to_notebook.py | 120 ++++++++++++++++++++++++ skills/README.md | 3 + skills/al_to_notebook.md | 47 ++++++++++ 5 files changed, 243 insertions(+) create mode 120000 .claude/skills/al_to_notebook.md create mode 100644 autoassistant/tests/test_to_notebook.py create mode 100644 autoassistant/to_notebook.py create mode 100644 skills/al_to_notebook.md diff --git a/.claude/skills/al_to_notebook.md b/.claude/skills/al_to_notebook.md new file mode 120000 index 0000000..bd9e5cb --- /dev/null +++ b/.claude/skills/al_to_notebook.md @@ -0,0 +1 @@ +../../skills/al_to_notebook.md \ No newline at end of file diff --git a/autoassistant/tests/test_to_notebook.py b/autoassistant/tests/test_to_notebook.py new file mode 100644 index 0000000..55819e4 --- /dev/null +++ b/autoassistant/tests/test_to_notebook.py @@ -0,0 +1,72 @@ +"""Unit tests for the narrative-docstring script → notebook converter. + +Fast and stdlib-only (json + tmp files): pins the cell-split semantics — docstring blocks +become markdown cells, code between becomes code cells — including the back-to-back +docstring case the PyAutoBuild reference pipeline mis-renders (two adjacent markdown +cells, never a code cell containing literal `# %%`/`'''` markers). +""" + +from __future__ import annotations + +import json + +from autoassistant.to_notebook import convert, script_to_cells + +SCRIPT = '''""" +Title +===== + +Intro prose with __Contents__. +""" + +import autolens as al + +""" +__Section One__ + +Explains the next code block. +""" + +tracer = None # code cell + +""" +__Section Two__ +""" + +""" +__Back To Back__ + +A second docstring immediately after the first — must stay markdown. +""" + +print("done") +''' + + +def test_cell_split_alternation(): + cells = script_to_cells(SCRIPT) + kinds = [c["cell_type"] for c in cells] + assert kinds == [ + "markdown", # title + "code", # import + "markdown", # section one + "code", # tracer + "markdown", # section two + "markdown", # back-to-back stays markdown + "code", # print + ] + assert "__Section One__" in "".join(cells[2]["source"]) + joined = "".join("".join(c["source"]) for c in cells) + assert "# %%" not in joined and "'''" not in joined + + +def test_convert_writes_valid_nbformat4(tmp_path): + script = tmp_path / "example.py" + script.write_text(SCRIPT, encoding="utf-8") + out = convert(script) + assert out == script.with_suffix(".ipynb") + nb = json.loads(out.read_text(encoding="utf-8")) + assert nb["nbformat"] == 4 + assert all("id" in c for c in nb["cells"]) + code = [c for c in nb["cells"] if c["cell_type"] == "code"] + assert all(c["outputs"] == [] and c["execution_count"] is None for c in code) diff --git a/autoassistant/to_notebook.py b/autoassistant/to_notebook.py new file mode 100644 index 0000000..99ed82d --- /dev/null +++ b/autoassistant/to_notebook.py @@ -0,0 +1,120 @@ +"""Convert a narrative-docstring PyAutoLens script to a Jupyter notebook. + +The assistant generates scripts in the workspace narrative-docstring style (an opening +title docstring with ``__Contents__``, then per-section ``\"\"\"__Section__\"\"\"`` +docstrings — ``skills/_style.md`` "Generated script style"). That style was adopted +precisely because notebook conversion is mechanical: + +- each **top-level docstring block** becomes a **markdown cell**; +- the **code between** docstring blocks becomes a **code cell**. + +This module is a self-contained, stdlib-only adaptation of the converter the PyAuto +workspaces already use at build time — ``PyAutoBuild:autobuild/build_util.py`` +(``py_to_notebook``) and ``PyAutoBuild:autobuild/add_notebook_quotes.py`` — which pipe +through the external ``ipynb-py-convert`` tool. It mirrors their cell-split semantics +(a line *starting* with triple quotes toggles docstring mode) but emits nbformat-v4 JSON +directly, so an assistant clone or a science project needs neither PyAutoBuild nor an +extra pip dependency. + +Usage: + + python -m autoassistant.to_notebook scripts/imaging.py # -> scripts/imaging.ipynb + python -m autoassistant.to_notebook scripts/imaging.py out.ipynb +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +_TRIPLE = ('"""', "'''") + + +def _cell(cell_type: str, lines: list[str]) -> dict: + """Build one nbformat-v4 cell; every source line keeps its newline except the last.""" + while lines and lines[-1].strip() == "": + lines.pop() + while lines and lines[0].strip() == "": + lines.pop(0) + source = [line + "\n" for line in lines] + if source: + source[-1] = source[-1].rstrip("\n") + cell: dict = {"cell_type": cell_type, "metadata": {}, "source": source} + if cell_type == "code": + cell.update({"execution_count": None, "outputs": []}) + return cell + + +def script_to_cells(text: str) -> list[dict]: + """Split narrative-docstring script text into alternating markdown/code cells.""" + cells: list[dict] = [] + buffer: list[str] = [] + in_docstring = False + + def flush(cell_type: str) -> None: + nonlocal buffer + if any(line.strip() for line in buffer): + cells.append(_cell(cell_type, buffer)) + buffer = [] + + for line in text.splitlines(): + stripped = line.lstrip() + starts = stripped.startswith(_TRIPLE) and stripped == line # top-level only + if starts and not in_docstring and len(line) > 6 and line.rstrip().endswith(line[:3]): + # Single-line docstring, e.g. """__Section__""" — one markdown cell. + flush("code") + cells.append(_cell("markdown", [line.strip()[3:-3]])) + continue + if starts: + flush("markdown" if in_docstring else "code") + in_docstring = not in_docstring + continue + buffer.append(line) + flush("markdown" if in_docstring else "code") + return cells + + +def convert(script_path: Path, notebook_path: Path | None = None) -> Path: + """Convert ``script_path`` to a notebook; returns the notebook path.""" + script_path = Path(script_path) + if notebook_path is None: + notebook_path = script_path.with_suffix(".ipynb") + cells = script_to_cells(script_path.read_text(encoding="utf-8")) + for i, cell in enumerate(cells): + cell["id"] = f"cell-{i}" + notebook = { + "cells": cells, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3", + }, + "language_info": {"name": "python"}, + }, + "nbformat": 4, + "nbformat_minor": 5, + } + Path(notebook_path).write_text( + json.dumps(notebook, indent=1, ensure_ascii=False) + "\n", encoding="utf-8" + ) + return Path(notebook_path) + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if not 1 <= len(args) <= 2: + print(__doc__.split("Usage:")[-1].strip(), file=sys.stderr) + return 1 + script = Path(args[0]) + if not script.is_file(): + print(f"not a file: {script}", file=sys.stderr) + return 1 + out = convert(script, Path(args[1]) if len(args) == 2 else None) + print(out.resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/README.md b/skills/README.md index 68eb438..d66ee8e 100644 --- a/skills/README.md +++ b/skills/README.md @@ -106,6 +106,9 @@ them. - [`al_inspect_source_reconstruction.md`](./al_inspect_source_reconstruction.md) — inspect a pixelised inversion: regularisation, source-plane image, reconstruction diagnostics. +- [`al_to_notebook.md`](./al_to_notebook.md) — convert a generated narrative-docstring + script to a Jupyter notebook (docstrings → markdown cells, code → code cells) via the + stdlib-only `autoassistant/to_notebook.py`. ### Pending — stubbed (need full recipes) diff --git a/skills/al_to_notebook.md b/skills/al_to_notebook.md new file mode 100644 index 0000000..80dfcfa --- /dev/null +++ b/skills/al_to_notebook.md @@ -0,0 +1,47 @@ +--- +name: al_to_notebook +description: Convert a generated narrative-docstring PyAutoLens script (.py) into a Jupyter notebook (.ipynb) — each top-level docstring becomes a markdown cell, the code between becomes code cells. Use when the user says "turn this into a notebook", "convert to ipynb", or wants a Jupyter/Colab version of a script the assistant produced. +user-invocable: true +--- + +# Script → notebook + +Generated scripts follow the workspace narrative-docstring style (`_style.md` "Generated +script style") precisely so this conversion is mechanical: each top-level `"""` docstring +block becomes a **markdown cell**; the Python between blocks becomes a **code cell**. + +## Orient + +The converter is `autoassistant/to_notebook.py` — stdlib-only, no external tools. It adapts +the converter the PyAuto workspaces use at build time (`PyAutoBuild:autobuild/build_util.py` +`py_to_notebook` + `add_notebook_quotes.py`) and mirrors its cell-split semantics. In a +science project, run it from the resolved assistant clone (refer-back). + +## Ask + +Nothing, usually — the script to convert is normally the one just produced. Ask only if +several candidate scripts are in play. + +## Branch — convert + +```bash +python -m autoassistant.to_notebook scripts/.py # -> scripts/.ipynb +python -m autoassistant.to_notebook scripts/.py .ipynb +``` + +(From a science project: `python /autoassistant/to_notebook.py …` works +identically.) The CLI prints the absolute output path — quote it and offer to open it. + +- The script must be in the narrative-docstring style; a script of bare comments converts to + one big code cell (correct, but probably not what the user wanted — say so). +- The notebook is **generated output**: never hand-edit the `.ipynb`; edit the `.py` and + reconvert. Keep the script the committed source of truth unless the user explicitly wants + the notebook tracked (e.g. a shared/Colab-facing artifact in a science project). + +## Combine + +- A converted notebook in a science project pairs well with the shareable-repo story + (`start-new-project.md` Collaborate/Publish): reviewers and collaborators often prefer + opening a notebook to running a script. +- For workspace-published notebooks (Colab setup cells, magic handling), the build pipeline + in PyAutoBuild remains authoritative — this skill is for assistant/project scripts.