From 70a9c60cad175b603af417500ecd6e9945b21786 Mon Sep 17 00:00:00 2001 From: madebysaira Date: Wed, 12 Aug 2026 22:53:46 +0530 Subject: [PATCH] feat: add word-timestamp auto-caption example --- examples/auto-captions/README.md | 49 +++++++++++ examples/auto-captions/auto_captions.py | 106 ++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 examples/auto-captions/README.md create mode 100644 examples/auto-captions/auto_captions.py diff --git a/examples/auto-captions/README.md b/examples/auto-captions/README.md new file mode 100644 index 0000000..c5a7bdc --- /dev/null +++ b/examples/auto-captions/README.md @@ -0,0 +1,49 @@ +# Auto-captions example + +This example converts word-timestamped speech-to-text output into FableCut `kind:"text"` clips with the built-in `karaoke` animation. It does not change the editor or require a specific speech-to-text engine. + +## 1. Produce word timestamps + +Provide a JSON file containing either a top-level `words` array or a bare array: + +```json +{ + "words": [ + {"word": "Welcome", "start": 0.00, "end": 0.42}, + {"word": "to", "start": 0.43, "end": 0.58}, + {"word": "FableCut", "start": 0.60, "end": 1.20} + ] +} +``` + +The format can be produced by faster-whisper, whisper.cpp adapters, or another STT engine. The script accepts `start` and `end` in seconds. + +## 2. Generate caption clips + +```bash +python examples/auto-captions/auto_captions.py \ + --transcript transcript.json \ + --output captions.json +``` + +The output contains a `clips` array. Add those clips to an existing `project.json` with the MCP `fablecut_patch_project` tool, or merge them into the project document before using `PUT /api/project`. + +```json +{"ops":[{"op":"addClip","clip":{"kind":"text","track":"V3","start":0,"duration":1.2,"props":{"text":"Welcome to FableCut","textAnim":"karaoke","wordRate":0.2}}}]} +``` + +The generated clips default to `V3`, use four words per line, cap a line at about 1.8 seconds, and derive `wordRate` from the observed word durations. Tune the grouping with `--max-words` and `--max-seconds`. + +## Optional local transcription + +Install faster-whisper separately, then let the script transcribe an audio or video file: + +```bash +pip install faster-whisper +python examples/auto-captions/auto_captions.py \ + --audio media/talk.mp4 \ + --model base \ + --output captions.json +``` + +The generated JSON is deliberately separate from `project.json`, so a user can review or transform the captions before applying them to a live editor. diff --git a/examples/auto-captions/auto_captions.py b/examples/auto-captions/auto_captions.py new file mode 100644 index 0000000..8b66954 --- /dev/null +++ b/examples/auto-captions/auto_captions.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Create FableCut karaoke text clips from word-timestamped transcription JSON. + +The input format is intentionally engine-agnostic: either a JSON file containing +{"words": [{"word": "hello", "start": 0.0, "end": 0.4}, ...]} or a list of +those word objects. Use --transcript to provide an existing transcript, or +--audio with faster-whisper installed to transcribe locally. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path +from typing import Any + + +def load_words(path: Path) -> list[dict[str, Any]]: + data = json.loads(path.read_text(encoding="utf-8")) + words = data.get("words", data) if isinstance(data, dict) else data + if not isinstance(words, list): + raise ValueError("transcript must be a list or an object containing a 'words' list") + clean = [] + for item in words: + if not isinstance(item, dict) or not item.get("word"): + continue + start, end = float(item["start"]), float(item["end"]) + if end <= start: + raise ValueError(f"word has invalid interval: {item!r}") + clean.append({"word": str(item["word"]).strip(), "start": start, "end": end}) + return sorted(clean, key=lambda item: item["start"]) + + +def transcribe(audio: Path, model_name: str) -> list[dict[str, Any]]: + try: + from faster_whisper import WhisperModel + except ImportError as exc: + raise SystemExit("--audio requires faster-whisper; install it with: pip install faster-whisper") from exc + model = WhisperModel(model_name) + segments, _ = model.transcribe(str(audio), word_timestamps=True) + return [ + {"word": word.word.strip(), "start": word.start, "end": word.end} + for segment in segments + for word in (segment.words or []) + if word.word.strip() + ] + + +def group_words(words: list[dict[str, Any]], max_words: int, max_seconds: float) -> list[dict[str, Any]]: + lines = [] + current: list[dict[str, Any]] = [] + for word in words: + too_many = len(current) >= max_words + too_long = current and word["end"] - current[0]["start"] > max_seconds + if current and (too_many or too_long): + lines.append(current) + current = [] + current.append(word) + if current: + lines.append(current) + return [ + { + "text": " ".join(word["word"] for word in line), + "start": line[0]["start"], + "duration": max(line[-1]["end"] - line[0]["start"], 0.01), + "wordRate": max((word["end"] - word["start"] for word in line), default=0.15), + } + for line in lines + ] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--transcript", type=Path, help="word-timestamp JSON file") + source.add_argument("--audio", type=Path, help="audio/video file; requires faster-whisper") + parser.add_argument("--model", default="base", help="faster-whisper model for --audio") + parser.add_argument("--output", type=Path, default=Path("project.captions.json")) + parser.add_argument("--max-words", type=int, default=4) + parser.add_argument("--max-seconds", type=float, default=1.8) + args = parser.parse_args() + words = load_words(args.transcript) if args.transcript else transcribe(args.audio, args.model) + clips = [] + for index, line in enumerate(group_words(words, args.max_words, args.max_seconds), 1): + clips.append({ + "id": f"caption_{index:04d}", + "kind": "text", + "track": "V3", + "start": line["start"], + "duration": line["duration"], + "props": { + "text": line["text"], + "textAnim": "karaoke", + "wordRate": line["wordRate"], + "fontSize": 72, + "bold": True, + "color": "#ffffff", + "textShadow": 12, + }, + }) + args.output.write_text(json.dumps({"clips": clips}, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {len(clips)} caption clips to {args.output}") + + +if __name__ == "__main__": + main()