diff --git a/docs/packages/core.mdx b/docs/packages/core.mdx index 22e41c7988..f581fb10df 100644 --- a/docs/packages/core.mdx +++ b/docs/packages/core.mdx @@ -372,6 +372,28 @@ The pre-built runtime IIFE is available as a direct import: import runtime from '@hyperframes/core/runtime'; ``` +### Audio Ducking + +Use `duck` from composition JavaScript to write deterministic volume ramps onto a GSAP timeline. The helper reads the timed media tracks you pass in and does not add any HTML authoring attributes: + +```typescript +import { duck } from '@hyperframes/core'; + +const timeline = gsap.timeline({ paused: true }); +const music = document.querySelector('#music'); +const voice = document.querySelector('#voice'); + +if (music && voice) { + duck(music, [voice], { + timeline, + amount: '-12dB', + fade: 0.3, + }); +} +``` + +To affect preview and render, pass the real DOM media elements as the tracks **and** the composition's captured GSAP timeline — those ramps are probed into the same volume envelope as hand-authored fades. Plain timing objects (`{ start, duration }`) and the returned keyframe array are for computing ducking offline (tests, custom pipelines); on their own they do not influence playback. `duck` writes its ramps in composition time, so it works the same for tracks placed at a non-zero `data-start`. + ## Frame Adapters The core package defines the [Frame Adapter](/concepts/frame-adapters) interface and provides the built-in GSAP adapter: diff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.ts index 1ceb5bc9d2..cac92cbed1 100644 --- a/packages/core/src/index.test.ts +++ b/packages/core/src/index.test.ts @@ -137,6 +137,10 @@ describe("@hyperframes/core public API exports", () => { it("exports parseAnimatedGifMetadata", () => { expect(typeof core.parseAnimatedGifMetadata).toBe("function"); }); + + it("exports duck", () => { + expect(typeof core.duck).toBe("function"); + }); }); describe("inline-script exports", () => { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c29992ba3d..34e96726e6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -219,6 +219,14 @@ export type { FitTextOptions, FitTextResult } from "./text/index.js"; // Runtime helpers (composition-side) export { getVariables } from "./runtime/getVariables.js"; +export { + duck, + type DuckAmount, + type DuckOptions, + type DuckTimelineLike, + type DuckTrack, + type DuckTrackTiming, +} from "./runtime/audioDucking.js"; // Variable validation (CLI / tooling-side) export { diff --git a/packages/core/src/runtime/audioDucking.test.ts b/packages/core/src/runtime/audioDucking.test.ts new file mode 100644 index 0000000000..0654e964c2 --- /dev/null +++ b/packages/core/src/runtime/audioDucking.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { + duck, + type DuckTimelineLike, + type DuckTrack, + type DuckTrackTiming, +} from "./audioDucking.js"; +import { interpolateVolumeGain, normaliseEnvelope } from "./mediaVolumeEnvelope.js"; + +interface TimelineOp { + kind: "set" | "to"; + target: DuckTrack; + at: number | undefined; + volume: number; + duration?: number; + ease?: "none"; + overwrite?: false; +} + +function recordingTimeline(ops: TimelineOp[]): DuckTimelineLike { + return { + set(target, vars, at) { + ops.push({ kind: "set", target, at, volume: vars.volume }); + }, + to(target, vars, at) { + ops.push({ + kind: "to", + target, + at, + volume: vars.volume, + duration: vars.duration, + ease: vars.ease, + overwrite: vars.overwrite, + }); + }, + }; +} + +function audioTrack(start: number, duration: number, volume = 1): DuckTrackTiming { + return { start, duration, volume }; +} + +function requireOp(ops: TimelineOp[], index: number): TimelineOp { + const op = ops[index]; + expect(op).toBeDefined(); + if (!op) throw new Error(`Missing timeline op ${index}`); + return op; +} + +describe("duck", () => { + it("generates volume ramps around voice overlaps", () => { + const keyframes = duck(audioTrack(0, 8, 0.8), audioTrack(2, 2), { + amount: "-12dB", + fade: 0.5, + }); + + expect(keyframes.map((kf) => kf.time)).toEqual([1.5, 2, 4, 4.5]); + expect(keyframes[0]?.volume).toBe(0.8); + expect(keyframes[1]?.volume).toBeCloseTo(0.20095, 5); + expect(keyframes[2]?.volume).toBeCloseTo(0.20095, 5); + expect(keyframes[3]?.volume).toBe(0.8); + }); + + it("writes generated ramps to a timeline", () => { + const music = audioTrack(0, 8, 0.8); + const voice = audioTrack(2, 2); + const ops: TimelineOp[] = []; + + duck(music, voice, { + timeline: recordingTimeline(ops), + amount: "-12dB", + fade: 0.5, + }); + + expect(ops).toHaveLength(4); + expect(requireOp(ops, 0)).toMatchObject({ kind: "set", target: music, at: 1.5, volume: 0.8 }); + expect(requireOp(ops, 1)).toMatchObject({ + kind: "to", + target: music, + at: 1.5, + duration: 0.5, + ease: "none", + overwrite: false, + }); + expect(requireOp(ops, 1).volume).toBeCloseTo(0.20095, 5); + expect(requireOp(ops, 2)).toMatchObject({ kind: "to", target: music, at: 2, duration: 2 }); + expect(requireOp(ops, 3)).toMatchObject({ + kind: "to", + target: music, + at: 4, + duration: 0.5, + volume: 0.8, + }); + }); + + it("merges voice gaps shorter than two fades", () => { + const keyframes = duck(audioTrack(0, 5), [audioTrack(1, 1), audioTrack(2.3, 0.7)], { + amount: 0.25, + fade: 0.25, + }); + + expect(keyframes).toEqual([ + { time: 0.75, volume: 1 }, + { time: 1, volume: 0.25 }, + { time: 3, volume: 0.25 }, + { time: 3.25, volume: 1 }, + ]); + }); + + it("uses resolved timeline duration for voice clips", () => { + const keyframes = duck(audioTrack(0, 8), audioTrack(1, 6), { + amount: 0.5, + fade: 0.5, + }); + + expect(keyframes).toEqual([ + { time: 0.5, volume: 1 }, + { time: 1, volume: 0.5 }, + { time: 7, volume: 0.5 }, + { time: 7.5, volume: 1 }, + ]); + }); + + it("produces keyframes compatible with the shared volume envelope", () => { + const keyframes = duck(audioTrack(0, 4), audioTrack(1, 1), { + amount: "-12dB", + fade: 0.25, + }); + const envelope = normaliseEnvelope(keyframes, 0, 1); + + expect(interpolateVolumeGain(envelope, 0.5)).toBe(1); + expect(interpolateVolumeGain(envelope, 1.5)).toBeCloseTo(0.251189, 5); + expect(interpolateVolumeGain(envelope, 2.5)).toBe(1); + }); + + it("reads timed media elements without writing schema attributes", () => { + const music = document.createElement("audio"); + music.id = "music"; + music.dataset.start = "0"; + music.dataset.duration = "4"; + music.dataset.volume = "0.6"; + + const voice = document.createElement("audio"); + voice.id = "voice"; + voice.dataset.start = "1"; + voice.dataset.duration = "1"; + + const keyframes = duck(music, [voice], { amount: 0.5, fade: 0.25 }); + + expect(keyframes).toEqual([ + { time: 0.75, volume: 0.6 }, + { time: 1, volume: 0.3 }, + { time: 2, volume: 0.3 }, + { time: 2.25, volume: 0.6 }, + ]); + expect(music.hasAttribute("data-duck")).toBe(false); + expect(music.hasAttribute("data-duck-fade")).toBe(false); + expect(music.hasAttribute("data-role")).toBe(false); + expect(music.getAttributeNames().some((name) => name.startsWith("data-hf-duck"))).toBe(false); + }); +}); diff --git a/packages/core/src/runtime/audioDucking.ts b/packages/core/src/runtime/audioDucking.ts new file mode 100644 index 0000000000..f6a5b54a46 --- /dev/null +++ b/packages/core/src/runtime/audioDucking.ts @@ -0,0 +1,290 @@ +import type { VolumeKeyframe } from "./mediaVolumeEnvelope.js"; + +export type DuckAmount = number | string; + +export interface DuckTimelineLike { + set: (target: DuckTrack, vars: { volume: number }, atSeconds?: number) => unknown; + to: ( + target: DuckTrack, + vars: { volume: number; duration: number; ease: "none"; overwrite: false }, + atSeconds?: number, + ) => unknown; +} + +export interface DuckOptions { + /** + * Duck amount as a linear gain (0.25) or dB string/negative number ("-12dB", -12). + * Defaults to -12 dB. + */ + amount?: DuckAmount; + /** Fade-in/fade-out ramp duration in seconds. Defaults to 0.3. */ + fade?: number; + /** GSAP-compatible paused timeline that receives the generated volume ramps. */ + timeline?: DuckTimelineLike; +} + +export interface DuckTrackTiming { + start: number; + end?: number; + duration?: number; + volume?: number; + muted?: boolean; + hasAudio?: boolean; +} + +export type DuckTrack = HTMLAudioElement | HTMLVideoElement | DuckTrackTiming; + +interface DuckInterval { + start: number; + end: number; +} + +interface ResolvedDuckTrack extends DuckInterval { + volume: number; + muted: boolean; + hasAudio: boolean; +} + +const DEFAULT_DUCK_AMOUNT = "-12dB"; +const DEFAULT_DUCK_FADE_SECONDS = 0.3; + +function clampVolume(volume: number): number { + if (!Number.isFinite(volume)) return 1; + return Math.max(0, Math.min(1, volume)); +} + +function finiteNumber(raw: string | number | null | undefined): number | null { + if (raw === null || raw === undefined) return null; + const parsed = typeof raw === "number" ? raw : Number.parseFloat(raw); + return Number.isFinite(parsed) ? parsed : null; +} + +function signedAmountToGain(value: number): number { + if (!Number.isFinite(value)) return 1; + return value < 0 ? Math.pow(10, value / 20) : value; +} + +function stringAmountToGain(raw: string): number { + const trimmed = raw.trim().toLowerCase(); + if (!trimmed) return 1; + + if (trimmed.endsWith("db")) { + const db = Number.parseFloat(trimmed.slice(0, -2)); + return Number.isFinite(db) ? Math.pow(10, db / 20) : 1; + } + + return signedAmountToGain(Number.parseFloat(trimmed)); +} + +function amountToGain(amount: DuckAmount | undefined): number { + const raw = amount ?? DEFAULT_DUCK_AMOUNT; + if (typeof raw === "number") return signedAmountToGain(raw); + return stringAmountToGain(raw); +} + +function timingEnd( + start: number, + end: number | undefined, + duration: number | undefined, +): number | null { + const explicitEnd = finiteNumber(end); + if (explicitEnd !== null) return explicitEnd; + const resolvedDuration = finiteNumber(duration); + return resolvedDuration === null ? null : start + resolvedDuration; +} + +function elementEnd(el: HTMLAudioElement | HTMLVideoElement, start: number): number | null { + const authoredEnd = finiteNumber(el.dataset.end); + if (authoredEnd !== null) return authoredEnd; + + const authoredDuration = finiteNumber(el.dataset.duration); + if (authoredDuration !== null) return start + authoredDuration; + + return Number.isFinite(el.duration) && el.duration > 0 ? start + el.duration : null; +} + +function isHtmlMediaTrack(track: DuckTrack): track is HTMLAudioElement | HTMLVideoElement { + return ( + typeof HTMLAudioElement !== "undefined" && + typeof HTMLVideoElement !== "undefined" && + (track instanceof HTMLAudioElement || track instanceof HTMLVideoElement) + ); +} + +function hasUsableEnd(start: number, end: number | null): end is number { + return end !== null && end > start; +} + +function resolveElementTrack(track: HTMLAudioElement | HTMLVideoElement): ResolvedDuckTrack | null { + const start = finiteNumber(track.dataset.start) ?? 0; + const end = elementEnd(track, start); + if (!hasUsableEnd(start, end)) return null; + const staticVolume = finiteNumber(track.dataset.volume); + return { + start, + end, + volume: clampVolume(staticVolume ?? track.volume), + muted: track.muted, + hasAudio: track instanceof HTMLAudioElement || track.dataset.hasAudio === "true", + }; +} + +function resolveTimingTrack(track: DuckTrackTiming): ResolvedDuckTrack | null { + const start = finiteNumber(track.start); + if (start === null) return null; + const end = timingEnd(start, track.end, track.duration); + if (!hasUsableEnd(start, end)) return null; + return { + start, + end, + volume: clampVolume(track.volume ?? 1), + muted: track.muted === true, + hasAudio: track.hasAudio !== false, + }; +} + +function resolveTrack(track: DuckTrack): ResolvedDuckTrack | null { + return isHtmlMediaTrack(track) ? resolveElementTrack(track) : resolveTimingTrack(track); +} + +function isAudible(track: ResolvedDuckTrack): boolean { + return track.hasAudio && !track.muted && track.volume > 0; +} + +function mergeIntervals(intervals: DuckInterval[], maxGap: number): DuckInterval[] { + const sorted = intervals + .filter((interval) => interval.end > interval.start) + .sort((a, b) => a.start - b.start); + const merged: DuckInterval[] = []; + + for (const interval of sorted) { + const previous = merged.at(-1); + if (previous && interval.start <= previous.end + maxGap) { + previous.end = Math.max(previous.end, interval.end); + } else { + merged.push({ ...interval }); + } + } + + return merged; +} + +function intersectIntervals(track: DuckInterval, intervals: DuckInterval[]): DuckInterval[] { + const overlaps: DuckInterval[] = []; + for (const interval of intervals) { + const start = Math.max(track.start, interval.start); + const end = Math.min(track.end, interval.end); + if (end > start) overlaps.push({ start, end }); + } + return overlaps; +} + +function roundedPoint(time: number, volume: number): VolumeKeyframe { + return { + time: Number(time.toFixed(6)), + volume: Number(clampVolume(volume).toFixed(6)), + }; +} + +function addPoint(keyframes: VolumeKeyframe[], time: number, volume: number): void { + const point = roundedPoint(time, volume); + const previous = keyframes.at(-1); + if (previous && Math.abs(previous.time - point.time) < 0.000001) { + previous.volume = point.volume; + } else { + keyframes.push(point); + } +} + +function appendDuckedWindow( + keyframes: VolumeKeyframe[], + music: ResolvedDuckTrack, + overlap: DuckInterval, + duckVolume: number, + fade: number, +): void { + const duration = overlap.end - overlap.start; + const rampDuration = Math.min(fade, duration / 2); + const rampStart = Math.max(music.start, overlap.start - rampDuration); + const rampEnd = Math.min(music.end, overlap.end + rampDuration); + + if (rampStart < overlap.start) addPoint(keyframes, rampStart, music.volume); + addPoint(keyframes, overlap.start, duckVolume); + addPoint(keyframes, overlap.end, duckVolume); + if (rampEnd > overlap.end) addPoint(keyframes, rampEnd, music.volume); +} + +function buildDuckKeyframes( + music: ResolvedDuckTrack, + voiceIntervals: DuckInterval[], + options: DuckOptions, +): VolumeKeyframe[] { + if (!isAudible(music)) return []; + + const fade = Math.max(0, finiteNumber(options.fade) ?? DEFAULT_DUCK_FADE_SECONDS); + const duckVolume = clampVolume(music.volume * amountToGain(options.amount)); + if (duckVolume >= music.volume - 0.000001) return []; + + const overlaps = mergeIntervals(intersectIntervals(music, voiceIntervals), fade * 2); + const keyframes: VolumeKeyframe[] = []; + for (const overlap of overlaps) appendDuckedWindow(keyframes, music, overlap, duckVolume, fade); + return keyframes; +} + +function resolveVoiceIntervals(voiceTracks: DuckTrack | DuckTrack[]): DuckInterval[] { + const tracks = Array.isArray(voiceTracks) ? voiceTracks : [voiceTracks]; + return mergeIntervals( + tracks + .map((track) => resolveTrack(track)) + .filter((track): track is ResolvedDuckTrack => track !== null && isAudible(track)) + .map((track) => ({ start: track.start, end: track.end })), + 0, + ); +} + +function writeKeyframes( + target: DuckTrack, + timeline: DuckTimelineLike | undefined, + keyframes: VolumeKeyframe[], +): void { + const [first, ...rest] = keyframes; + if (!timeline || !first) return; + + timeline.set(target, { volume: first.volume }, first.time); + let previous = first; + + for (const current of rest) { + const duration = current.time - previous.time; + if (duration <= 0.000001) { + timeline.set(target, { volume: current.volume }, current.time); + } else { + timeline.to( + target, + { volume: current.volume, duration, ease: "none", overwrite: false }, + previous.time, + ); + } + previous = current; + } +} + +/** + * Programmatically author deterministic music ducking. + * + * The helper computes overlap windows from already-authored media timings and + * writes linear volume ramps onto the supplied GSAP-compatible timeline. The + * producer's existing volume-automation probe then turns those timeline writes + * into the same render-time volume keyframes used by hand-authored fades. + */ +export function duck( + musicTrack: DuckTrack, + voiceTracks: DuckTrack | DuckTrack[], + options: DuckOptions = {}, +): VolumeKeyframe[] { + const music = resolveTrack(musicTrack); + if (!music) return []; + + const keyframes = buildDuckKeyframes(music, resolveVoiceIntervals(voiceTracks), options); + writeKeyframes(musicTrack, options.timeline, keyframes); + return keyframes; +} diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts index bd15db9942..1532270ea7 100644 --- a/packages/core/src/runtime/media.test.ts +++ b/packages/core/src/runtime/media.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect, vi, afterEach } from "vitest"; import { readElementPlaybackRate, refreshRuntimeMediaCache, syncRuntimeMedia } from "./media"; import type { RuntimeMediaClip } from "./media"; +import { duck } from "./audioDucking"; +import { interpolateVolumeGain, normaliseEnvelope } from "./mediaVolumeEnvelope"; function createVideo(attrs: Record): HTMLVideoElement { const el = document.createElement("video"); @@ -943,4 +945,61 @@ describe("syncRuntimeMedia", () => { }); expect(clip.el.muted).toBe(true); }); + + describe("volume keyframes (preview/render parity)", () => { + // Regression for the duck()/probe parity bug: probed keyframes carry ABSOLUTE + // composition-time stamps, and the renderer normalises them by the track start + // before baking, so its envelope lookup is shift-invariant in composition time. + // Preview must look the envelope up at absolute composition time too. The + // earlier code interpolated absolute keyframes at the track-relative `relTime`, + // which missed the duck entirely for any track with a non-zero data-start. + const baseVolume = 1; + // Music at comp-time 5s..13s, voice at 6s..7s → duck keyframes around + // 5.75 / 6 / 7 / 7.25 (all ABSOLUTE composition seconds). + const keyframes = duck( + { start: 5, duration: 8, volume: baseVolume }, + { start: 6, duration: 1 }, + { + amount: "-12dB", + fade: 0.25, + }, + ); + + // The renderer's basis: normalise by trackStart, then look up in + // track-relative seconds. Equivalent (by shift-invariance) to interpolating + // the raw absolute keyframes at absolute composition time, which is what the + // fixed preview path now does. + const renderGainAt = (compTime: number): number => + interpolateVolumeGain(normaliseEnvelope(keyframes, 5, baseVolume), compTime - 5); + + it("ducks during voice overlap on a non-zero data-start track", () => { + const clip = createMockClip({ start: 5, end: 13, volume: baseVolume }); + clip.volumeKeyframes = keyframes; + // Middle of the duck window (comp-time 6.5s). The buggy relTime lookup + // (relTime = 1.5) sampled before the first absolute keyframe (5.75) and + // returned the un-ducked base volume; the fixed path ducks here. + syncRuntimeMedia({ clips: [clip], timeSeconds: 6.5, playing: true, playbackRate: 1 }); + expect(clip.el.volume).toBeCloseTo(0.251189, 5); + expect(clip.el.volume).toBeCloseTo(renderGainAt(6.5), 6); + expect(clip.el.volume).toBeLessThan(baseVolume); // would be 1.0 under the bug + }); + + it("holds base volume before the duck ramp begins", () => { + const clip = createMockClip({ start: 5, end: 13, volume: baseVolume }); + clip.volumeKeyframes = keyframes; + // Comp-time 5.2s — the clip is active but the ramp (5.75s) has not started. + syncRuntimeMedia({ clips: [clip], timeSeconds: 5.2, playing: true, playbackRate: 1 }); + expect(clip.el.volume).toBeCloseTo(baseVolume, 5); + expect(clip.el.volume).toBeCloseTo(renderGainAt(5.2), 6); + }); + + it("matches the renderer envelope across the active window", () => { + const clip = createMockClip({ start: 5, end: 13, volume: baseVolume }); + clip.volumeKeyframes = keyframes; + for (const compTime of [5.0, 5.75, 6.0, 6.5, 7.0, 7.25, 8.0]) { + syncRuntimeMedia({ clips: [clip], timeSeconds: compTime, playing: true, playbackRate: 1 }); + expect(clip.el.volume).toBeCloseTo(renderGainAt(compTime), 6); + } + }); + }); }); diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index aa8e1b71da..e1c0334549 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -192,8 +192,18 @@ export function syncRuntimeMedia(params: { let authorVolume: number; if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) { // Keyframes probed from the GSAP timeline — same source as the renderer. - // Use the interpolated envelope value directly; no need to track GSAP changes. - authorVolume = clampVolume(interpolateVolumeGain(clip.volumeKeyframes, relTime)); + // They carry ABSOLUTE composition-time stamps (the probe samples + // el.volume at absolute timeline seconds, and duck() writes its ramps at + // absolute overlap times). The renderer normalises them by the track + // start before baking (engine/audioVolumeEnvelope.ts → normaliseEnvelope) + // and then looks the envelope up in track-relative seconds, which makes + // its result shift-invariant in composition time. Preview must use the + // SAME basis, so look the envelope up at absolute composition time — NOT + // relTime. relTime is track-relative AND folds in mediaStart/playbackRate + // (which the envelope was never authored against), so interpolating + // absolute keyframes at relTime reads the wrong window and misses the + // duck for any track with a non-zero data-start. + authorVolume = clampVolume(interpolateVolumeGain(clip.volumeKeyframes, params.timeSeconds)); } else if (previousRuntimeVolume === undefined) { // First tick this clip is active. The transport has already seeked GSAP // to the current time (seekTimelineAndAdapters runs before syncRuntimeMedia),