From e70c710015f27fcb4b5837d90c4c3a121bd7a7ff Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 6 May 2026 19:41:20 +0000 Subject: [PATCH] feat(pipeline): derive Manim/VHS stages from visual_map; demos playwright dogfood generate-all now renders only Manim scenes and VHS tapes referenced by active segments in visual_map, so playwright_test segments use pre-recorded video without running capture stages. Wire docs/demos with discover_tests, narration_from_source, catalog init file, segment 07 (playwright_test + checked-in WebM), README, and milestone checklist updates. Extend gitignore to allow the committed dogfood WebM under terminal/rendered/. Co-authored-by: John Menke --- .gitignore | 3 +- docgen.catalog.yaml | 5 ++ docs/demos/README.md | 60 ++++++++++++++++++ docs/demos/docgen.yaml | 35 ++++++++++ docs/demos/narration/07-playwright-dogfood.md | 3 + .../rendered/07-playwright-dogfood.webm | Bin 0 -> 5426 bytes .../checklist-playwright-auto-narration.md | 4 +- milestones/next-session-dogfood.md | 18 +++--- milestones/upstream-dogfood.md | 2 +- src/docgen/config.py | 34 ++++++++++ src/docgen/manim_runner.py | 24 +++++-- src/docgen/pipeline.py | 42 ++++++++---- src/docgen/vhs.py | 16 ++++- tests/test_config.py | 39 ++++++++++++ tests/test_pipeline.py | 6 +- 15 files changed, 257 insertions(+), 34 deletions(-) create mode 100644 docgen.catalog.yaml create mode 100644 docs/demos/README.md create mode 100644 docs/demos/narration/07-playwright-dogfood.md create mode 100644 docs/demos/terminal/rendered/07-playwright-dogfood.webm diff --git a/.gitignore b/.gitignore index 47f1b57..8651b09 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,8 @@ htmlcov/ # Generated demo assets (reproducible via docgen generate-all) docs/demos/audio/*.mp3 docs/demos/animations/media/ -docs/demos/terminal/rendered/ +docs/demos/terminal/rendered/* +!docs/demos/terminal/rendered/07-playwright-dogfood.webm docs/demos/.docgen-state.json # recordings/*.mp4 — COMMITTED via Git LFS for GitHub Pages diff --git a/docgen.catalog.yaml b/docgen.catalog.yaml new file mode 100644 index 0000000..354487c --- /dev/null +++ b/docgen.catalog.yaml @@ -0,0 +1,5 @@ +catalog_schema_version: 1 +updated_at: '2026-05-06T19:38:58Z' +docgen_version: 0.2.0 +repo_root: . +entries: [] diff --git a/docs/demos/README.md b/docs/demos/README.md new file mode 100644 index 0000000..5e06875 --- /dev/null +++ b/docs/demos/README.md @@ -0,0 +1,60 @@ +# docgen demo videos (dogfood) + +This tree is the **in-repo dogfood** project: the documentation-generator repository eating its own cooking. Configuration lives in `docgen.yaml` (paths relative to this directory unless noted). + +## Prerequisites + +Full `docgen generate-all` needs **Manim**, **ffmpeg**, **VHS**, **ttyd**, **Xvfb** (or a display) for terminal tapes, **Playwright** only if you re-record browser footage, and **`OPENAI_API_KEY`** for Whisper timestamps, TTS, and optional `narration-generate`. + +Segment **07** uses a **checked-in WebM** under `terminal/rendered/`; no Playwright run is required for a basic pipeline pass if you keep that file. + +## Commands (typical) + +From repository root: + +```bash +cd docs/demos +docgen --config docgen.yaml validate --help +``` + +Source-catalog metadata (repo root; shared with other workflows): + +```bash +cd docs/demos +docgen --config docgen.yaml catalog init # once; creates ../../docgen.catalog.yaml +docgen --config docgen.yaml catalog stale +``` + +Discovery and catalog merge (Node Playwright projects under the repo root): + +```bash +cd docs/demos +docgen --config docgen.yaml discover-tests --merge-catalog +docgen --config docgen.yaml catalog refresh +``` + +Optional narration draft for segment 07 (requires `OPENAI_API_KEY`): + +```bash +cd docs/demos +docgen --config docgen.yaml narration-generate --segment 07 --dry-run +docgen --config docgen.yaml narration-generate --segment 07 +``` + +Full pipeline (heavy): + +```bash +cd docs/demos +docgen --config docgen.yaml generate-all +# or iterate with skips, e.g. --skip-tts after audio exists +``` + +Validation: + +```bash +cd docs/demos +docgen --config docgen.yaml validate +docgen --config docgen.yaml validate --pre-push +``` + +Upstream consumer dogfood (separate clone) is described in `milestones/upstream-dogfood.md` at the repo root. diff --git a/docs/demos/docgen.yaml b/docs/demos/docgen.yaml index 921a136..dd6a3db 100644 --- a/docs/demos/docgen.yaml +++ b/docs/demos/docgen.yaml @@ -1,5 +1,6 @@ # docgen.yaml — dogfood: docgen's own demo videos # See: https://github.com/jmjava/documentation-generator +# Command cheat sheet: README.md in this directory. # # This project uses docgen to produce demo videos about docgen itself. # docgen generate-all # full pipeline @@ -9,6 +10,30 @@ repo_root: ../.. env_file: ../../.env +discover_tests: + roots: + - "." + +narration_from_source: + model: gpt-4o-mini + hints: + - > + Project context: docgen is the documentation-generator CLI/library (see AGENTS.md). + Audience: engineers adopting docgen or reviewing its pipeline. + context: + paths: + - README.md + - AGENTS.md + - src/docgen/pipeline.py + - src/docgen/compose.py + - src/docgen/config.py + segments: + "07": + hints: + - > + This segment demonstrates visual_map type playwright_test with a short pre-recorded + WebM under terminal/rendered/. Keep the tone consistent with other demo segments. + dirs: narration: narration audio: audio @@ -24,6 +49,7 @@ segments: - "04" - "05" - "06" + - "07" all: *all_segments segment_names: @@ -33,10 +59,12 @@ segment_names: "04": 04-tts-pipeline "05": 05-compose-validate "06": 06-ci-integration + "07": 07-playwright-dogfood # Visual source mapping # 01, 03: Manim animations (architecture overview, wizard GUI walkthrough) # 02, 04, 05, 06: VHS terminal recordings (live docgen commands) +# 07: playwright_test — pre-recorded WebM (dogfood; pipeline skips Manim/VHS capture for this id) visual_map: "01": type: manim @@ -62,6 +90,10 @@ visual_map: type: vhs tape: 06-ci-integration.tape source: 06-ci-integration.mp4 + "07": + type: playwright_test + test: dogfood/playwright_mux_placeholder.spec.ts + source: 07-playwright-dogfood.webm manim: quality: 720p30 @@ -129,6 +161,9 @@ pages: "06": title: "CI Integration" description: "Pre-push hooks and GitHub Actions for continuous validation." + "07": + title: "Playwright visual mux (dogfood)" + description: "Pre-recorded browser-style footage composed like Manim/VHS segments." wizard: llm_model: gpt-4o diff --git a/docs/demos/narration/07-playwright-dogfood.md b/docs/demos/narration/07-playwright-dogfood.md new file mode 100644 index 0000000..2ce984e --- /dev/null +++ b/docs/demos/narration/07-playwright-dogfood.md @@ -0,0 +1,3 @@ +This segment shows how docgen treats browser-test footage like any other visual source. You record or export a short Playwright video, point the manifest at that file, and compose muxes it with the narration audio from this repository. + +The pipeline skips Manim and VHS capture for this segment because the WebM is already on disk. That keeps long-running UI tests out of the default generate-all path while still proving the playwright_test visual type end to end. diff --git a/docs/demos/terminal/rendered/07-playwright-dogfood.webm b/docs/demos/terminal/rendered/07-playwright-dogfood.webm new file mode 100644 index 0000000000000000000000000000000000000000..7414e99666aa55105772fa1c68650d962d725e39 GIT binary patch literal 5426 zcmcJTUr1Y59LLYSNz66IKO3{8BVf8eZII4h*Wt8%=rtx(#b{`(+Onb3&DeGufmD~r zdlIW@-KcH%Ae}Hvs3}CI)e-BVC)IYijfk-cZ$J#{Hc#b>~J(YQF0YYQIsE)RL4CJysX@ zkf!Y6zewd9PDj0icsuQj&z`qFK7O*j*79S??*XF(XU~S~r}v$KBm0-@TPMGq3cmT? zxNWkbxKwfV|L5cX=1$I1;wA$`yFN++k}m59fA~3(nQ=ba8jbnLZ+#wO~Y z1O2O<7wlhFFu$yFUU2`&ZJ=N0ykI}_1n6G^^KW8)lZ)}dd`-2WKaAx!je!0X=LP#W ztzmw15tiRv3Hm!YFS!5B3D7^ycwoNf4Cc2G&I|T$afAK<=LN4{OBD2vabB>0OB(cN zvHUp>%%7t%9++=V59l8T^Us0)Rn80cZ&fhARmJjK+d#k0dBOc}odEqyj0fgx-NgKg zi}Qm0E483M%z44eyzm0HSuz#By z^arr~wkYTyx>8HYo7r9OJM#@%%AV#ykP(NwV*$Y< str: def visual_map(self) -> dict[str, Any]: return self.raw.get("visual_map", {}) + def pipeline_manim_scene_names(self) -> list[str]: + """Scene class names for ``segments.all`` entries whose ``visual_map`` type is ``manim``.""" + seen: set[str] = set() + ordered: list[str] = [] + for seg_id in self.segments_all: + vm = self.visual_map.get(seg_id) + if not isinstance(vm, dict): + continue + if str(vm.get("type", "")).lower() != "manim": + continue + scene = str(vm.get("scene", "")).strip() + if scene and scene not in seen: + seen.add(scene) + ordered.append(scene) + return ordered + + def pipeline_vhs_tape_filenames(self) -> list[str]: + """Tape filenames for ``segments.all`` entries whose ``visual_map`` type is ``vhs``.""" + ordered: list[str] = [] + for seg_id in self.segments_all: + vm = self.visual_map.get(seg_id) + if not isinstance(vm, dict): + continue + if str(vm.get("type", "")).lower() != "vhs": + continue + tape = str(vm.get("tape", "")).strip() + if not tape: + src = str(vm.get("source", "")).strip() + if src: + tape = f"{Path(src).stem}.tape" + if tape: + ordered.append(tape) + return ordered + @property def concat_map(self) -> dict[str, list[str]]: return self.raw.get("concat", {}) diff --git a/src/docgen/manim_runner.py b/src/docgen/manim_runner.py index 8abe71b..9f989fa 100644 --- a/src/docgen/manim_runner.py +++ b/src/docgen/manim_runner.py @@ -17,9 +17,25 @@ class ManimRunner: def __init__(self, config: Config) -> None: self.config = config - def render(self, scene: str | None = None) -> None: - scenes = [scene] if scene else self.config.manim_scenes - if not scenes: + def render( + self, + scenes: list[str] | None = None, + *, + scene: str | None = None, + ) -> None: + """Render Manim scenes. + + * ``scene=`` — single scene (CLI / wizard). + * ``scenes=`` — explicit list (pipeline uses :meth:`Config.pipeline_manim_scene_names`). + * Otherwise — ``config.manim`` ``scenes:`` list (legacy). + """ + if scene is not None: + to_render = [scene] + elif scenes is not None: + to_render = scenes + else: + to_render = self.config.manim_scenes + if not to_render: print("[manim] No scenes configured") return @@ -37,7 +53,7 @@ def render(self, scene: str | None = None) -> None: font = self.config.manim_font print(f"[manim] Rendering at {quality_label}, font={font}") - for s in scenes: + for s in to_render: self._render_one(manim_bin, scenes_file, s, quality_args) def _check_font(self) -> None: diff --git a/src/docgen/pipeline.py b/src/docgen/pipeline.py index 8dd26b0..f8ab8de 100644 --- a/src/docgen/pipeline.py +++ b/src/docgen/pipeline.py @@ -1,4 +1,10 @@ -"""Pipeline orchestrator: tts -> manim -> vhs -> compose -> validate -> concat -> pages.""" +"""Pipeline orchestrator: tts -> manim -> vhs -> compose -> validate -> concat -> pages. + +Manim and VHS stages render only scenes/tapes referenced by ``visual_map`` for active +``segments.all`` entries (see :meth:`docgen.config.Config.pipeline_manim_scene_names` and +:meth:`~docgen.config.Config.pipeline_vhs_tape_filenames`). Segments whose visuals are +``playwright_test`` use pre-recorded files and do not run through Manim or VHS capture here. +""" from __future__ import annotations @@ -36,17 +42,25 @@ def run( TapeSynchronizer(self.config).sync() if not skip_manim: - print("\n=== Stage: Manim ===") - from docgen.manim_runner import ManimRunner - ManimRunner(self.config).render() + scene_list = self.config.pipeline_manim_scene_names() + if scene_list: + print("\n=== Stage: Manim ===") + from docgen.manim_runner import ManimRunner + ManimRunner(self.config).render(scenes=scene_list) + else: + print("\n=== Stage: Manim (skipped — no manim segments in visual_map) ===") if not skip_vhs: - print("\n=== Stage: VHS ===") - from docgen.vhs import VHSRunner - results = VHSRunner(self.config).render() - for r in results: - if not r.success: - print(f" WARNING: {r.tape} had errors: {r.errors}") + tape_list = self.config.pipeline_vhs_tape_filenames() + if tape_list: + print("\n=== Stage: VHS ===") + from docgen.vhs import VHSRunner + results = VHSRunner(self.config).render(tapes=tape_list) + for r in results: + if not r.success: + print(f" WARNING: {r.tape} had errors: {r.errors}") + else: + print("\n=== Stage: VHS (skipped — no vhs segments in visual_map) ===") print("\n=== Stage: Compose ===") from docgen.compose import ComposeError, Composer @@ -57,9 +71,11 @@ def run( if self._should_retry_manim(exc, skip_manim, retry_manim_on_freeze): print("\n=== Compose FREEZE GUARD detected; retrying Manim + compose once ===") self._clear_manim_media_cache() - print("\n=== Stage: Manim (retry) ===") - from docgen.manim_runner import ManimRunner - ManimRunner(self.config).render() + scene_list = self.config.pipeline_manim_scene_names() + if scene_list: + print("\n=== Stage: Manim (retry) ===") + from docgen.manim_runner import ManimRunner + ManimRunner(self.config).render(scenes=scene_list) print("\n=== Stage: Compose (retry) ===") composer.compose_segments(self.config.segments_all) else: diff --git a/src/docgen/vhs.py b/src/docgen/vhs.py index 54ac8de..f08cc6e 100644 --- a/src/docgen/vhs.py +++ b/src/docgen/vhs.py @@ -118,6 +118,8 @@ def render( tape: str | None = None, strict: bool = False, timeout_sec: int | None = None, + *, + tapes: list[str] | None = None, ) -> list[VHSResult]: terminal_dir = self.config.terminal_dir if not terminal_dir.exists(): @@ -125,11 +127,19 @@ def render( return [] if tape: - tapes = [terminal_dir / tape] if (terminal_dir / tape).exists() else list( + tapes_paths = [terminal_dir / tape] if (terminal_dir / tape).exists() else list( terminal_dir.glob(f"*{tape}*") ) + elif tapes is not None: + tapes_paths = [] + for name in tapes: + path = terminal_dir / name + if path.exists(): + tapes_paths.append(path) + else: + tapes_paths.extend(sorted(terminal_dir.glob(f"*{name}*"))) else: - tapes = sorted(terminal_dir.glob("*.tape")) + tapes_paths = sorted(terminal_dir.glob("*.tape")) results: list[VHSResult] = [] effective_timeout = timeout_sec @@ -139,7 +149,7 @@ def render( if self.render_timeout_sec is not None else self.config.vhs_render_timeout_sec ) - for t in tapes: + for t in tapes_paths: results.append(self._render_one(t, strict, effective_timeout)) return results diff --git a/tests/test_config.py b/tests/test_config.py index 6ca121a..34b3cec 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -213,3 +213,42 @@ def test_discover_tests_scan_roots_monorepo(tmp_path): (tmp_path / "docgen.yaml").write_text(yaml.dump(cfg), encoding="utf-8") c = Config.from_yaml(tmp_path / "docgen.yaml") assert c.discover_tests_scan_roots == [tmp_path.resolve(), apps.resolve()] + + +def test_pipeline_manim_scene_names_from_visual_map(tmp_path): + cfg = { + "segments": {"all": ["01", "07", "03"]}, + "visual_map": { + "01": {"type": "manim", "scene": "OverviewScene"}, + "03": {"type": "manim", "scene": "WizardScene"}, + "07": {"type": "playwright_test", "test": "e/x.spec.ts", "source": "v.webm"}, + }, + } + (tmp_path / "docgen.yaml").write_text(yaml.dump(cfg), encoding="utf-8") + c = Config.from_yaml(tmp_path / "docgen.yaml") + assert c.pipeline_manim_scene_names() == ["OverviewScene", "WizardScene"] + + +def test_pipeline_vhs_tape_filenames_from_visual_map(tmp_path): + cfg = { + "segments": {"all": ["02", "07"]}, + "visual_map": { + "02": {"type": "vhs", "tape": "02-init.tape", "source": "02-init.mp4"}, + "07": {"type": "playwright_test", "test": "e/x.spec.ts", "source": "v.webm"}, + }, + } + (tmp_path / "docgen.yaml").write_text(yaml.dump(cfg), encoding="utf-8") + c = Config.from_yaml(tmp_path / "docgen.yaml") + assert c.pipeline_vhs_tape_filenames() == ["02-init.tape"] + + +def test_pipeline_vhs_tape_derives_from_source_when_tape_missing(tmp_path): + cfg = { + "segments": {"all": ["02"]}, + "visual_map": { + "02": {"type": "vhs", "source": "foo-bar.mp4"}, + }, + } + (tmp_path / "docgen.yaml").write_text(yaml.dump(cfg), encoding="utf-8") + c = Config.from_yaml(tmp_path / "docgen.yaml") + assert c.pipeline_vhs_tape_filenames() == ["foo-bar.tape"] diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 5b4e4f2..2c9b4df 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -22,7 +22,7 @@ class FakeManimRunner: def __init__(self, _config) -> None: pass - def render(self, scene=None) -> None: + def render(self, scenes=None, *, scene=None) -> None: calls.append("manim") class FakeValidator: @@ -92,6 +92,8 @@ def compose_segments(self, _segments) -> int: animations_dir=animations_dir, segments_all=["01"], sync_vhs_after_timestamps=False, + pipeline_manim_scene_names=lambda: ["Scene01"], + pipeline_vhs_tape_filenames=lambda: [], ) Pipeline(cfg).run(skip_tts=True, skip_vhs=True, retry_manim_on_freeze=True) @@ -122,6 +124,8 @@ def compose_segments(self, _segments) -> int: animations_dir=animations_dir, segments_all=["01"], sync_vhs_after_timestamps=False, + pipeline_manim_scene_names=lambda: ["Scene01"], + pipeline_vhs_tape_filenames=lambda: [], ) with pytest.raises(ComposeError, match="FREEZE GUARD"):