From 5652ce46fd2e123f289e0c20a43f91bc17b3dbb6 Mon Sep 17 00:00:00 2001 From: Dario Lencina Date: Thu, 9 Jul 2026 23:31:33 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20[SFX:]=20timeline=20lines=20=E2=80=94?= =?UTF-8?q?=20mix=20sound=20effects=20at=20absolute=20offsets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [SFX: assets/sfx/impact.mp3 | at=5.8 gain=-15] inside a scene block places a sound effect at scene.start + at, volume-adjusted in dB. SFX enter the assemble amix graph like narration clips (adelay + normalize) but never get atempo and never gate scene durations. A missing file fails assembly loudly — an authored cue that ships as silence is a bug. Co-Authored-By: Claude Fable 5 --- crates/videoeditor-media/src/assemble.rs | 24 ++++++++++++++ crates/videoeditor-timeline/src/lib.rs | 40 +++++++++++++++++++++++- crates/videoeditor/guide.md | 6 ++++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/crates/videoeditor-media/src/assemble.rs b/crates/videoeditor-media/src/assemble.rs index fe4350f..9c0348e 100644 --- a/crates/videoeditor-media/src/assemble.rs +++ b/crates/videoeditor-media/src/assemble.rs @@ -76,6 +76,30 @@ pub fn run(ep: &Episode) -> Result<()> { } } + // sound effects — placed like narration (absolute offset), no tempo. + // A missing file is an authoring typo: fail loudly, don't ship silence. + for scene in &ep.scenes { + for sfx in &scene.sfx { + let src = ep.root.join(&sfx.file); + if !src.exists() { + bail!("sfx {} not found (scene `{}`)", src.display(), scene.name); + } + let abs_ms = ((scene.start + sfx.at) * 1000.0).round() as i64; + let idx = inputs.len(); + inputs.push(src.display().to_string()); + let label = format!("s{idx}"); + let gain = if sfx.gain_db.abs() > 1e-6 { + format!("volume={}dB,", sfx.gain_db) + } else { + String::new() + }; + chains.push(format!( + "[{idx}:a]{gain}{norm},adelay={abs_ms}:all=1[{label}]" + )); + mix_labels.push(label); + } + } + // native audio from video-clip scenes for scene in &ep.scenes { if !scene.is_video_clip() { diff --git a/crates/videoeditor-timeline/src/lib.rs b/crates/videoeditor-timeline/src/lib.rs index 2bd0fcf..0f37465 100644 --- a/crates/videoeditor-timeline/src/lib.rs +++ b/crates/videoeditor-timeline/src/lib.rs @@ -16,6 +16,7 @@ //! //! [SCENE: name | template=code-meme duration=6.42] //! [DATA: code=assets/code/threads.rs lang=rust bench="μ: 150µS|σ: 50µS" bench_at=5.8] +//! [SFX: assets/sfx/impact.mp3 | at=5.8 gain=-15] //! [CLIP: threads | at=0.19] //! Narration text until the next marker. //! ``` @@ -75,6 +76,7 @@ pub struct Scene { pub start: f64, pub data: Map, pub clips: Vec, + pub sfx: Vec, } #[derive(Debug, Serialize)] @@ -86,6 +88,18 @@ pub struct Clip { pub tempo: f64, } +/// A sound effect placed on the timeline (`[SFX: file | at=sec gain=db]`). +/// Mixed at `scene.start + at`, gain in dB (0 = as authored). Unlike +/// narration, SFX never gate scene durations and never get tempo applied. +#[derive(Debug, Serialize)] +pub struct Sfx { + /// Audio file path, relative to the episode root. + pub file: String, + /// Offset from scene start, seconds. + pub at: f64, + pub gain_db: f64, +} + /// One generated narration clip, as recorded in `audio/clips.json`. /// Written by the TTS stage, consumed by assembly. #[derive(Debug, Serialize, Deserialize)] @@ -392,6 +406,7 @@ fn parse_scenes(body: &str) -> Result> { start: 0.0, data, clips: vec![], + sfx: vec![], }); } else if let Some((_, attrs)) = parse_marker(trimmed, "DATA") { let scene = scenes.last_mut().context("[DATA:] before any [SCENE:]")?; @@ -407,8 +422,21 @@ fn parse_scenes(body: &str) -> Result> { at: attr_f64(&attrs, "at").ok(), tempo: attr_f64(&attrs, "tempo").unwrap_or(1.0), }); + } else if let Some((file, attrs)) = parse_marker(trimmed, "SFX") { + // deliberately no flush_clip: an [SFX:] between narration lines + // must not split the surrounding clip's text + let scene = scenes.last_mut().context("[SFX:] before any [SCENE:]")?; + scene.sfx.push(Sfx { + at: attr_f64(&attrs, "at").with_context(|| format!("sfx `{file}` missing at="))?, + gain_db: if attrs.iter().any(|(k, _)| k == "gain") { + attr_f64(&attrs, "gain").with_context(|| format!("sfx `{file}`"))? + } else { + 0.0 + }, + file, + }); } else if trimmed.starts_with('[') && trimmed.ends_with(']') { - // unknown marker ([IMAGE:], [SFX:], comments) — ignored, not narrated + // unknown marker ([IMAGE:], comments) — ignored, not narrated } else if !trimmed.is_empty() && !trimmed.starts_with('#') { clip_text.push(trimmed.to_string()); } @@ -532,8 +560,10 @@ X versus Y for TOPIC. [SCENE: good | template=code-meme duration=6.4] [DATA: code=assets/code/good.rs bench="μ: 150µS|σ: 50µS" flat=true] +[SFX: assets/sfx/impact.mp3 | at=5.8 gain=-15] [CLIP: explain | at=0.2 tempo=1.05] First line. +[SFX: assets/sfx/whoosh.mp3 | at=1.2] Second line joins the same clip. [SCENE: outro | template=video-clip duration=2.2] @@ -579,6 +609,14 @@ Second line joins the same clip. "First line. Second line joins the same clip." ); + // [SFX:] lines attach to the scene without splitting clip text + assert_eq!(good.sfx.len(), 2); + assert_eq!(good.sfx[0].file, "assets/sfx/impact.mp3"); + assert_eq!(good.sfx[0].at, 5.8); + assert_eq!(good.sfx[0].gain_db, -15.0); + assert_eq!(good.sfx[1].file, "assets/sfx/whoosh.mp3"); + assert_eq!(good.sfx[1].gain_db, 0.0); + let outro = &scenes[2]; assert!(outro.is_video_clip()); assert_eq!(outro.data.get("audio"), Some(&Value::Bool(false))); diff --git a/crates/videoeditor/guide.md b/crates/videoeditor/guide.md index 124af75..8b1043b 100644 --- a/crates/videoeditor/guide.md +++ b/crates/videoeditor/guide.md @@ -43,6 +43,7 @@ Narration text until the next marker. [SCENE: body | template=code-meme duration=7.0] [DATA: code=assets/code/x.rs code_size=34 bench="μ: 1.2ms" bench_at=5.5] +[SFX: assets/sfx/impact.mp3 | at=5.5 gain=-15] [CLIP: body | at=0.2] One idea per beat. The screen holds the digits; the voice tells the story. ``` @@ -50,6 +51,11 @@ One idea per beat. The screen holds the digits; the voice tells the story. Scene `duration` is authoritative. `at` = seconds from scene start. Paths are episode-relative. `` and unknown `[MARKERS:]` are ignored. +`[SFX:]` mixes a sound effect at `scene.start + at` — no tempo, never gates +scene durations. `gain` is dB (default 0); keep effects ≈ −15 dB so narration +always dominates, and land impacts in narration pauses, never over a spoken +number. + ## The director loop (in order, every time) 1. **Script** per the craft rules below.