diff --git a/.changeset/capture-data-audio-producer.md b/.changeset/capture-data-audio-producer.md new file mode 100644 index 0000000..8154971 --- /dev/null +++ b/.changeset/capture-data-audio-producer.md @@ -0,0 +1,8 @@ +--- +'@vosjs/core': minor +--- + +capture templates gain runtime-input and audio capabilities: + +- `capture.data` — JSON-injected into the page and passed to `initVos` as `deps.data` (capture-video AND capture-thumbnail). Data-dependent compositions (constant program + inputs in `ctx.data`) now render correctly in capture modes instead of falling back to baked config data. +- `capture.audioProducerCode` (capture-video) — host-supplied JavaScript defining `window.__vosAudioProducer__ = async ({ data, duration, sampleRate }) => AudioBuffer | null`; the template calls it and muxes the returned buffer as the output's audio track (AAC for mp4 with automatic Opus fallback where AAC encode is unavailable, Opus for webm). The engine imposes no audio schema — producers interpret `data` however the host defines. Without a producer, zero audio code is emitted. diff --git a/packages/core/src/__tests__/renderTemplate.test.ts b/packages/core/src/__tests__/renderTemplate.test.ts index 0ab4e45..fd7bac2 100644 --- a/packages/core/src/__tests__/renderTemplate.test.ts +++ b/packages/core/src/__tests__/renderTemplate.test.ts @@ -263,3 +263,75 @@ describe('generateRenderTemplate', () => { }) }) }) + +describe('capture-video: data injection and audio producer', () => { + const capture = { width: 640, height: 360, duration: 5, fps: 30 } + + it('injects capture.data as deps.data (video and thumbnail modes)', () => { + const video = generateRenderTemplate(sampleCode, { + mode: 'capture-video', + capture: { ...capture, data: { videoSrc: 'x.webm', micGain: 0.8 } }, + }) + expect(video).toContain('"videoSrc":"x.webm"') + expect(video).toContain('if (__captureData != null) deps.data = __captureData;') + + const thumb = generateRenderTemplate(sampleCode, { + mode: 'capture-thumbnail', + capture: { ...capture, data: { videoSrc: 'x.webm' } }, + }) + expect(thumb).toContain('"videoSrc":"x.webm"') + expect(thumb).toContain('deps.data = __captureData') + }) + + it('defaults to null data (compositions fall back to baked config data)', () => { + const html = generateRenderTemplate(sampleCode, { + mode: 'capture-video', + capture, + }) + expect(html).toContain('const __captureData = null;') + }) + + it('embeds the audio producer and muxes its buffer when provided', () => { + const html = generateRenderTemplate(sampleCode, { + mode: 'capture-video', + capture: { + ...capture, + audioProducerCode: + 'window.__vosAudioProducer__ = async () => null // host mixer', + }, + }) + expect(html).toContain('window.__vosAudioProducer__ = async () => null') + expect(html).toContain('AudioBufferSource') + expect(html).toContain('addAudioTrack') + }) + + it('prefers AAC for mp4 with an Opus fallback, Opus for webm', () => { + const mp4 = generateRenderTemplate(sampleCode, { + mode: 'capture-video', + capture: { + ...capture, + format: 'mp4' as const, + audioProducerCode: 'window.__vosAudioProducer__ = async () => null', + }, + }) + expect(mp4).toContain('const preferred = "aac"') + expect(mp4).toContain('canEncodeAudio') + const webm = generateRenderTemplate(sampleCode, { + mode: 'capture-video', + capture: { + ...capture, + audioProducerCode: 'window.__vosAudioProducer__ = async () => null', + }, + }) + expect(webm).toContain('const preferred = "opus"') + }) + + it('emits no audio plumbing without a producer', () => { + const html = generateRenderTemplate(sampleCode, { + mode: 'capture-video', + capture, + }) + expect(html).not.toContain('__vosAudioProducer__') + expect(html).not.toContain('addAudioTrack') + }) +}) diff --git a/packages/core/src/runtime/renderTemplate.ts b/packages/core/src/runtime/renderTemplate.ts index 5a9190d..7ff2fce 100644 --- a/packages/core/src/runtime/renderTemplate.ts +++ b/packages/core/src/runtime/renderTemplate.ts @@ -64,6 +64,26 @@ export interface RenderTemplateOptions { * through strings. */ uploadUrl?: string + /** + * Passed to `initVos(container, deps)` as `deps.data` (JSON-serialized + * into the page). Compositions that read `ctx.data` at runtime — the + * interpreter pattern, where the program is constant and inputs travel + * as data — need this to render correctly in capture modes; without it + * they fall back to the config's baked `data` (often empty). + */ + data?: Record + /** + * Host-supplied audio producer (capture-video mode): JavaScript source + * evaluated in the page that must define + * `window.__vosAudioProducer__ = async ({ data, duration, sampleRate }) + * => AudioBuffer | null`. When set, the template calls it and muxes the + * returned buffer as the output's audio track (AAC for mp4 with an + * Opus fallback — AAC encode is unavailable on some platforms — and + * Opus for webm). Returning null (or omitting the option) captures + * video-only. The engine imposes no audio schema — how the producer + * interprets `data` is entirely the host's concern. + */ + audioProducerCode?: string } /** URLs to generate hints for */ preloadModuleUrls?: string[] @@ -623,17 +643,63 @@ function generateCaptureVideoBody( });` const mimeType = isMp4 ? 'video/mp4' : 'video/webm' + const dataJson = capture.data ? JSON.stringify(capture.data) : 'null' + const audioProducerBlock = capture.audioProducerCode + ? ` + // Host-supplied audio producer (defines window.__vosAudioProducer__) + ${capture.audioProducerCode}` + : '' + + // Audio plumbing is emitted only when a producer is supplied — pages + // without one carry zero audio code. + const audioSetup = capture.audioProducerCode + ? ` + // Host audio (before the frame loop so a producer failure aborts + // early instead of wasting the render). Null buffer = video-only. + const audioBuffer = window.__vosAudioProducer__ + ? await window.__vosAudioProducer__({ + data: __captureData, + duration: ${duration}, + sampleRate: 48000, + }) + : null; + let audioSource = null; + if (audioBuffer) { + const { AudioBufferSource, canEncodeAudio } = await import('mediabunny'); + // AAC first for mp4 (compatibility), Opus fallback — AAC + // encode is unavailable on some platforms (e.g. Linux). + const preferred = ${JSON.stringify(isMp4 ? 'aac' : 'opus')}; + const audioCodec = + preferred === 'aac' && !(await canEncodeAudio('aac')) ? 'opus' : preferred; + audioSource = new AudioBufferSource({ codec: audioCodec, bitrate: QUALITY_HIGH }); + output.addAudioTrack(audioSource); + } +` + : '' + const audioFeed = capture.audioProducerCode + ? ` + if (audioSource && audioBuffer) { + await audioSource.add(audioBuffer); + audioSource.close(); + } +` + : '' + return ` // Make THREE and gsap available globally for animation code window.THREE = THREE; window.gsap = gsap; ${elementsBlock} +${audioProducerBlock} // Deterministic capture: compositions must SEEK their videos, never // play them (same contract as the client-side exporter). window.__vos__.isPaused = true; + // Runtime inputs for data-dependent compositions (ctx.data). + const __captureData = ${dataJson}; + // Override window dimensions Object.defineProperty(window, 'innerWidth', { value: ${width}, configurable: true }); Object.defineProperty(window, 'innerHeight', { value: ${height}, configurable: true }); @@ -660,6 +726,7 @@ ${elementsBlock} drawingBufferHeight: ${height} } }; + if (__captureData != null) deps.data = __captureData; // Initialize animation const initFn = window.initVos || window.initAnimation; @@ -693,7 +760,9 @@ ${elementsBlock} ${formatSetup} output.addVideoTrack(videoSource, { frameRate: ${fps} }); +${audioSetup} await output.start(); +${audioFeed} // Frames [startFrame, endFrame) — evaluated at global composition // time, captured with segment-local timestamps (the file starts @@ -792,6 +861,7 @@ function generateCaptureThumbnailBody( ): string { const { width, height } = capture const thumbnailTime = capture.thumbnailTime ?? 0.5 + const dataJson = capture.data ? JSON.stringify(capture.data) : 'null' const transformedCode = transformModuleCode(animationCode, 'server') return ` @@ -827,6 +897,8 @@ ${elementsBlock} drawingBufferHeight: ${height} } }; + const __captureData = ${dataJson}; + if (__captureData != null) deps.data = __captureData; // Initialize animation const initFn = window.initVos || window.initAnimation;