feat(voice): local speech stack by default — whisper.cpp STT + piper TTS

CLAUDE.md

@@ -12,7 +12,11 @@ vertical video out via headless Chrome + ffmpeg + ElevenLabs).
 - `crates/videoeditor-chrome` — CDP driver (long-lived headless Chrome; NEVER
   single-shot `--screenshot`, it hangs on macOS).
 - `crates/videoeditor-media` — all ffmpeg/ffprobe invocations + assembly.
-- `crates/videoeditor-voice` — ElevenLabs TTS/STT (`ELEVENLABS_API_KEY`).
+- `crates/videoeditor-voice` — TTS/STT with two backends each: local
+  default (piper voice via sherpa-onnx; whisper.cpp) and ElevenLabs
+  (`ELEVENLABS_API_KEY`; `tts:` frontmatter / `VIDEOEDITOR_TTS` /
+  `VIDEOEDITOR_STT` select). Local models come from the nix closure
+  (`WHISPER_MODEL`, `PIPER_VOICE`).
 - `crates/videoeditor-genai` — typed image-generation clients: xAI Grok
   Imagine (`XAI_API_KEY`, reference images) + Google Imagen (`AI_STUDIO`);
   Veo/Grok video is the planned next tenant.

README.md

@@ -10,7 +10,9 @@ from markdown + SVG placeholders. This strip is actual output.*
 - **Rust** orchestrates everything (`videoeditor`, one binary).
 - **Web tech** does the animation: scenes are HTML templates rendered
   frame-by-frame by headless Chrome as pure functions of `(data, t)`.
-- **ffmpeg** does the heavy lifting; **ElevenLabs** voices the narration.
+- **ffmpeg** does the heavy lifting; **local speech models** voice the
+  narration and transcribe (piper + whisper.cpp, no API key), with
+  ElevenLabs as an opt-in upgrade.
 - **Built to be driven by [Claude Code](https://claude.com/claude-code)** —
   scaffolds wire the session up automatically.
 
@@ -26,8 +28,9 @@ from markdown + SVG placeholders. This strip is actual output.*
 ## Install
 
 With [nix](https://install.determinate.systems) (preferred — pins the binary,
-ffmpeg, and the render browser from the committed lockfile; details and every
-run variant in [docs/nix.md](docs/nix.md)):
+ffmpeg, the render browser, and the local speech stack (whisper.cpp STT +
+a piper voice) from the committed lockfile; details and every run variant
+in [docs/nix.md](docs/nix.md)):
 
 ```bash
 curl -fsSL https://install.determinate.systems/nix | sh -s -- install   # once
@@ -38,9 +41,12 @@ Without nix: `cargo install videoeditor`, then bring ffmpeg
 (`brew/apt/dnf install ffmpeg`) and Chrome (system install is auto-detected;
 `CHROME_BIN` overrides). macOS and Linux; on Windows use WSL.
 
-Voicing needs an `ELEVENLABS_API_KEY`
+Everything runs keyless by default: narration uses a bundled local piper
+voice, transcription uses bundled whisper.cpp. Prefer ElevenLabs voices?
+Set `tts: elevenlabs` in the script frontmatter with an `ELEVENLABS_API_KEY`
 ([elevenlabs.io](https://elevenlabs.io) → profile → API keys; free tier is
-plenty). Everything except `tts`/`analyze` runs keyless.
+plenty). Non-nix installs bring their own whisper.cpp/sherpa-onnx for the
+local stack (see `videoeditor guide`, Env section).
 
 ## Your first video
 
@@ -72,7 +78,7 @@ anything costs money. Or skip the wizard and ask in your own words:
 ```
 script.md ──parse──► timeline plan
    │
-   ├─ videoeditor tts       [CLIP:] → ElevenLabs → audio/clips/ + audio/clips.json
+   ├─ videoeditor tts       [CLIP:] → piper (or ElevenLabs) → audio/clips/ + audio/clips.json
    ├─ videoeditor render    [SCENE:] → Chrome frames → ffmpeg → build/scenes/
    └─ videoeditor assemble  concat + narration@offsets + music → build/final.mp4
 ```

crates/videoeditor-media/src/lib.rs

@@ -147,6 +147,36 @@ pub fn extract_audio(video: &Path, out: &Path) -> Result<()> {
     ])
 }
 
+/// Downmix any audio to 16 kHz mono PCM wav — the input whisper.cpp expects.
+pub fn to_whisper_wav(audio: &Path, out: &Path) -> Result<()> {
+    ffmpeg(&[
+        "-i",
+        audio.to_str().unwrap(),
+        "-ar",
+        "16000",
+        "-ac",
+        "1",
+        "-c:a",
+        "pcm_s16le",
+        out.to_str().unwrap(),
+    ])
+}
+
+/// Encode a wav (e.g. piper TTS output) to the pipeline's mp3 format.
+pub fn wav_to_mp3(wav: &Path, out: &Path) -> Result<()> {
+    ffmpeg(&[
+        "-i",
+        wav.to_str().unwrap(),
+        "-ar",
+        "44100",
+        "-codec:a",
+        "libmp3lame",
+        "-b:a",
+        "128k",
+        out.to_str().unwrap(),
+    ])
+}
+
 /// Detect scene cuts with ffmpeg's `select=gt(scene,threshold)` filter;
 /// returns cut timestamps in seconds.
 pub fn scene_cuts(video: &Path, threshold: f32) -> Result<Vec<f64>> {

crates/videoeditor-record/src/lib.rs

@@ -393,7 +393,8 @@ struct Review {
     mean_db: f64,
     max_db: f64,
     clipped: bool,
-    /// None = no ELEVENLABS_API_KEY; level/timing coaching still runs.
+    /// None = no STT backend available (whisper binary+model, or
+    /// ELEVENLABS_API_KEY); level/timing coaching still runs.
     transcript: Option<String>,
     accuracy_pct: Option<f64>,
     missing: Vec<String>,
@@ -407,8 +408,8 @@ struct Review {
 /// Archive and analyze a pending take WITHOUT keeping it: the take lands
 /// in `audio/takes/<id>/` permanently (data safety — retakes lose
 /// nothing), then local level metrics via ffmpeg always; script-accuracy
-/// / pacing / dead-air / background-noise coaching via ElevenLabs Scribe
-/// when a key is present.
+/// / pacing / dead-air / background-noise coaching via STT (whisper by
+/// default, ElevenLabs Scribe via VIDEOEDITOR_STT) when available.
 fn review_take(ep: &Episode, id: &str, body: &[u8], mime: &str) -> Result<Review> {
     let (scene, clip) = find_clip(ep, id)?;
     let n = store_take(ep, id, body, mime)?;
@@ -420,9 +421,9 @@ fn review_take(ep: &Episode, id: &str, body: &[u8], mime: &str) -> Result<Review
     let (mean_db, max_db) = audio_levels(&mp3)?;
     let clipped = max_db > -0.2;
 
-    // Scribe is optional — no key, no transcript coaching.
-    let stt = if videoeditor_voice::api_key().is_ok() {
-        Some(videoeditor_voice::stt(&mp3).context("ElevenLabs STT")?)
+    // STT is optional — no backend, no transcript coaching.
+    let stt = if videoeditor_voice::stt_available() {
+        Some(videoeditor_voice::stt(&mp3).context("speech-to-text")?)
     } else {
         None
     };

crates/videoeditor-timeline/src/lib.rs

@@ -9,7 +9,8 @@
 //! ---
 //! title: My Episode
 //! fps: 30
-//! voice_id: pNInz6obpgDQGcFmaJgB   # "Adam" — an ElevenLabs public preset
+//! tts: piper                       # narration backend: piper (local, default) | elevenlabs
+//! voice_id: pNInz6obpgDQGcFmaJgB   # elevenlabs only — "Adam", a public preset
 //! music: assets/music/bed.mp3
 //! ---
 //!
@@ -52,6 +53,9 @@ pub struct Meta {
     /// Template packs this episode uses (frontmatter `packs:`,
     /// comma-separated paths relative to the episode dir).
     pub packs: Vec<String>,
+    /// Narration backend (frontmatter `tts:`): "piper" | "elevenlabs".
+    /// None = unset; the voice crate resolves the default (piper).
+    pub tts: Option<String>,
     pub voice_id: Option<String>,
     pub model_id: String,
     /// ElevenLabs voice_settings — low stability reads livelier, less robotic.
@@ -285,6 +289,7 @@ fn parse_meta(front: &str) -> Result<Meta> {
     let mut width = 1080u32;
     let mut height = 1920u32;
     let mut packs = Vec::new();
+    let mut tts = None;
     let mut voice_id = None;
     let mut model_id = "eleven_multilingual_v2".to_string();
     let mut voice_stability = 0.4;
@@ -310,6 +315,7 @@ fn parse_meta(front: &str) -> Result<Meta> {
                     .filter(|p| !p.is_empty())
                     .collect()
             }
+            "tts" => tts = Some(v),
             "voice_id" => voice_id = Some(v),
             "model_id" => model_id = v,
             "voice_stability" => voice_stability = v.parse().context("voice_stability")?,
@@ -326,6 +332,7 @@ fn parse_meta(front: &str) -> Result<Meta> {
         width,
         height,
         packs,
+        tts,
         voice_id,
         model_id,
         voice_stability,
@@ -510,6 +517,7 @@ mod tests {
     const SCRIPT: &str = r#"---
 title: Demo
 fps: 30
+tts: elevenlabs
 voice_id: pNInz6obpgDQGcFmaJgB
 music: assets/music/bed.mp3
 ---
@@ -544,6 +552,7 @@ Second line joins the same clip.
         assert_eq!(meta.title, "Demo");
         assert_eq!(meta.fps, 30);
         assert_eq!(meta.width, 1080);
+        assert_eq!(meta.tts.as_deref(), Some("elevenlabs"));
         assert_eq!(meta.voice_id.as_deref(), Some("pNInz6obpgDQGcFmaJgB"));
         assert_eq!(meta.music.as_deref(), Some("assets/music/bed.mp3"));
     }

crates/videoeditor-voice/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "videoeditor-voice"
-description = "ElevenLabs TTS and STT client for videoeditor: narration synthesis and reference transcription"
-keywords = ["tts", "stt", "elevenlabs", "narration", "video"]
+description = "Voice I/O for videoeditor: local piper TTS + whisper.cpp STT (default) or ElevenLabs, narration synthesis and reference transcription"
+keywords = ["tts", "stt", "whisper", "narration", "video"]
 categories = ["multimedia::audio", "api-bindings"]
 version.workspace = true
 edition.workspace = true

crates/videoeditor-voice/examples/transcribe.rs

@@ -0,0 +1,20 @@
+//! Transcribe one audio file with the resolved STT backend and print the
+//! normalized transcript JSON:
+//!
+//! ```sh
+//! cargo run -p videoeditor-voice --example transcribe -- take.mp3
+//! ```
+
+use anyhow::{Context, Result};
+use std::path::PathBuf;
+
+fn main() -> Result<()> {
+    let audio: PathBuf = std::env::args_os()
+        .nth(1)
+        .context("usage: transcribe <audio-file>")?
+        .into();
+    eprintln!("stt backend: {}", videoeditor_voice::stt_name());
+    let transcript = videoeditor_voice::stt(&audio)?;
+    println!("{}", serde_json::to_string_pretty(&transcript)?);
+    Ok(())
+}

crates/videoeditor-voice/src/elevenlabs.rs

@@ -0,0 +1,100 @@
+//! ElevenLabs cloud backend: TTS (voice_id presets, mp3 out) and Scribe STT.
+
+use anyhow::{Context, Result, bail};
+use serde_json::Value;
+use std::env;
+use std::fs;
+use std::io::Read;
+use std::path::Path;
+use videoeditor_timeline::Meta;
+
+pub fn api_key() -> Result<String> {
+    env::var("ELEVENLABS_API_KEY")
+        .or_else(|_| env::var("ELEVENLAB"))
+        .context("set ELEVENLABS_API_KEY (your ElevenLabs API key)")
+}
+
+pub fn synth(meta: &Meta, text: &str, out: &Path) -> Result<()> {
+    let key = api_key()?;
+    let voice = meta
+        .voice_id
+        .as_deref()
+        .context("frontmatter needs voice_id: for ElevenLabs TTS")?;
+    let url =
+        format!("https://api.elevenlabs.io/v1/text-to-speech/{voice}?output_format=mp3_44100_128");
+    let resp = ureq::post(&url)
+        .set("xi-api-key", &key)
+        .send_json(serde_json::json!({
+            "text": text,
+            "model_id": meta.model_id,
+            "voice_settings": {
+                "stability": meta.voice_stability,
+                "similarity_boost": meta.voice_similarity,
+                "style": meta.voice_style,
+                "use_speaker_boost": true
+            }
+        }));
+    let resp = match resp {
+        Ok(r) => r,
+        Err(ureq::Error::Status(code, r)) => {
+            bail!(
+                "ElevenLabs TTS {code}: {}",
+                r.into_string().unwrap_or_default()
+            )
+        }
+        Err(e) => return Err(e.into()),
+    };
+    let mut bytes = Vec::new();
+    resp.into_reader().read_to_end(&mut bytes)?;
+    fs::write(out, bytes)?;
+    Ok(())
+}
+
+/// Transcribe with ElevenLabs Scribe (word-level timestamps). Returns the
+/// Scribe response as-is — it is already the crate's transcript shape.
+pub fn stt(audio: &Path) -> Result<Value> {
+    let key = api_key()?;
+    let bytes = fs::read(audio)?;
+    let boundary = "----videoeditorboundary7d1c9a2f";
+    let mut body = Vec::new();
+    for (name, value) in [
+        ("model_id", "scribe_v1"),
+        ("timestamps_granularity", "word"),
+    ] {
+        body.extend_from_slice(
+            format!(
+                "--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
+            )
+            .as_bytes(),
+        );
+    }
+    body.extend_from_slice(
+        format!(
+            "--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.mp3\"\r\nContent-Type: audio/mpeg\r\n\r\n"
+        )
+        .as_bytes(),
+    );
+    body.extend_from_slice(&bytes);
+    body.extend_from_slice(format!("\r\n--{boundary}--\r\n").as_bytes());
+
+    let resp = ureq::post("https://api.elevenlabs.io/v1/speech-to-text")
+        .set("xi-api-key", &key)
+        .set(
+            "content-type",
+            &format!("multipart/form-data; boundary={boundary}"),
+        )
+        .send_bytes(&body);
+    let resp = match resp {
+        Ok(r) => r,
+        Err(ureq::Error::Status(code, r)) => {
+            bail!(
+                "ElevenLabs STT {code}: {}",
+                r.into_string().unwrap_or_default()
+            )
+        }
+        Err(e) => return Err(e.into()),
+    };
+    let mut s = String::new();
+    resp.into_reader().read_to_string(&mut s)?;
+    Ok(serde_json::from_str(&s)?)
+}