From d925abac09a0a41de99bf96f38026d367feaa3f0 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:32:42 -0700 Subject: [PATCH 01/56] Scaffold livekit-capture crate --- Cargo.toml | 2 ++ livekit-capture/Cargo.toml | 14 ++++++++++++++ livekit-capture/src/lib.rs | 15 +++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 livekit-capture/Cargo.toml create mode 100644 livekit-capture/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index f23e9eb1e..867dc39ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "livekit-uniffi", "livekit-datatrack", "livekit-ffi-node-bindings", + "livekit-capture", "livekit-runtime", "livekit-wakeword", "libwebrtc", @@ -50,6 +51,7 @@ imgproc = { version = "0.3.19", path = "imgproc" } libwebrtc = { version = "0.3.41", path = "libwebrtc" } livekit = { version = "0.7.52", path = "livekit" } livekit-api = { version = "0.5.5", path = "livekit-api" } +livekit-capture = { version = "0.1.0", path = "livekit-capture" } livekit-ffi = { version = "0.12.70", path = "livekit-ffi" } livekit-datatrack = { version = "0.1.11", path = "livekit-datatrack" } livekit-common = { version = "0.1.0", path = "livekit-common" } diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml new file mode 100644 index 000000000..edf9f9991 --- /dev/null +++ b/livekit-capture/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "livekit-capture" +description = "Encoded video ingest helpers for LiveKit" +version = "0.1.0" +readme = "README.md" +license.workspace = true +edition.workspace = true +repository.workspace = true + +[dependencies] +bytes = { workspace = true } +livekit = { workspace = true } +log = { workspace = true } +thiserror = { workspace = true } diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs new file mode 100644 index 000000000..226bbce09 --- /dev/null +++ b/livekit-capture/src/lib.rs @@ -0,0 +1,15 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Helpers for publishing pre-encoded video with LiveKit. From e34ce632e39ef74ee79531021d779b925874df83 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:33:30 -0700 Subject: [PATCH 02/56] Add error module to livekit-capture --- livekit-capture/src/error.rs | 38 ++++++++++++++++++++++++++++++++++++ livekit-capture/src/lib.rs | 4 ++++ 2 files changed, 42 insertions(+) create mode 100644 livekit-capture/src/error.rs diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs new file mode 100644 index 000000000..344299d24 --- /dev/null +++ b/livekit-capture/src/error.rs @@ -0,0 +1,38 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use thiserror::Error; + +/// Error returned by capture helpers. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum CaptureError { + /// Encoded payload is empty. + #[error("encoded payload is empty")] + EmptyPayload, + /// H.265 NAL unit is too short to contain its header. + #[error("H.265 NAL unit is too short")] + H265NalTooShort, + /// Access unit carries layering metadata the passthrough cannot forward. + #[error("unsupported layered encoding: {0}")] + UnsupportedLayeredEncoding(&'static str), + /// Encoded payload or transport data is malformed. + #[error("invalid encoded data: {0}")] + InvalidEncodedData(&'static str), + /// Capture backend is not available on this platform. + #[error("{0} is not supported on this platform")] + UnsupportedPlatform(&'static str), + /// The underlying source rejected the frame. + #[error("capture source rejected the frame")] + CaptureFailed, +} diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 226bbce09..05b2405e0 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -13,3 +13,7 @@ // limitations under the License. //! Helpers for publishing pre-encoded video with LiveKit. + +mod error; + +pub use error::CaptureError; From 670cb60ef027e8cf87eaae40b357741e1aac2591 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:41:45 -0700 Subject: [PATCH 03/56] Add encoded access unit types --- livekit-capture/src/encoded.rs | 532 +++++++++++++++++++++++++++++++++ livekit-capture/src/error.rs | 8 + livekit-capture/src/lib.rs | 6 + 3 files changed, 546 insertions(+) create mode 100644 livekit-capture/src/encoded.rs diff --git a/livekit-capture/src/encoded.rs b/livekit-capture/src/encoded.rs new file mode 100644 index 000000000..f6b428ab8 --- /dev/null +++ b/livekit-capture/src/encoded.rs @@ -0,0 +1,532 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use livekit::{ + options::VideoCodec, + webrtc::video_frame::{ + EncodedFrameType as RtcEncodedFrameType, EncodedVideoCodec as RtcEncodedVideoCodec, + }, +}; + +use crate::error::CaptureError; + +const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; + +/// Encoder rate-control target requested by WebRTC for an encoded source. +pub use livekit::webrtc::video_source::EncodedRateControl; + +/// Encoded byte-stream framing used by encoded source backends. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum EncodedWireFormat { + /// H.264 Annex-B byte stream. + H264AnnexB, + /// H.264/AVC byte stream with length-prefixed NAL units. + /// + /// `nal_length_size` is the number of big-endian length bytes before each NAL unit. Values + /// from 1 through 4 are accepted; 4 is the common AVC configuration. + H264Avc { + /// Length-prefix size in bytes. + nal_length_size: u8, + }, + /// H.265 Annex-B byte stream. + H265AnnexB, + /// RTP packets for the supplied codec and RTP clock rate. + Rtp { + /// RTP payload codec. + codec: EncodedVideoCodec, + /// RTP timestamp clock rate. + clock_rate: u32, + }, + /// MPEG transport stream carrying encoded video. + MpegTs, +} + +/// Encoded video codec carried by an [`EncodedAccessUnit`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum EncodedVideoCodec { + /// H.264/AVC video. + H264, + /// H.265/HEVC video. + H265, + /// VP8 video. + VP8, + /// VP9 video. + VP9, + /// AV1 video. + AV1, +} + +/// Encoded video frame type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncodedFrameType { + /// A key frame. + Key, + /// A delta frame. + Delta, +} + +/// Layer identifiers associated with an encoded frame. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct EncodedLayerInfo { + /// Spatial layer index, when present. + pub spatial_id: Option, + /// Temporal layer index, when present. + pub temporal_id: Option, +} + +/// H.264 packetization mode for passthrough metadata. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum H264PacketizationMode { + /// Non-interleaved packetization mode. + NonInterleaved, +} + +/// Codec-specific metadata for encoded passthrough. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum CodecSpecific { + /// No codec-specific metadata. + None, + /// H.264-specific metadata. + H264 { + /// H.264 RTP packetization mode. + packetization_mode: H264PacketizationMode, + }, + /// H.265-specific metadata. + H265, + /// VP8-specific metadata. + VP8 { + /// Temporal layer index, when present. + temporal_id: Option, + /// Whether this frame synchronizes a temporal layer. + layer_sync: bool, + }, + /// VP9-specific metadata. + VP9 { + /// Temporal layer index, when present. + temporal_id: Option, + /// Spatial layer index, when present. + spatial_id: Option, + /// Whether this frame depends on an inter-layer reference. + inter_layer_predicted: Option, + }, + /// AV1-specific metadata. + AV1 { + /// RTP scalability mode, such as `L1T1`. + scalability_mode: Option, + /// Encoded dependency descriptor bytes, when supplied by the caller. + dependency_descriptor: Option>, + }, +} + +impl Default for CodecSpecific { + fn default() -> Self { + Self::None + } +} + +impl CodecSpecific { + /// Returns the single-layer default metadata for a codec, matching what + /// the passthrough encoder synthesizes on the wire. + pub fn default_for(codec: EncodedVideoCodec) -> Self { + match codec { + EncodedVideoCodec::H264 => { + Self::H264 { packetization_mode: H264PacketizationMode::NonInterleaved } + } + EncodedVideoCodec::H265 => Self::H265, + EncodedVideoCodec::VP8 => Self::VP8 { temporal_id: None, layer_sync: false }, + EncodedVideoCodec::VP9 => { + Self::VP9 { temporal_id: None, spatial_id: None, inter_layer_predicted: None } + } + EncodedVideoCodec::AV1 => { + Self::AV1 { scalability_mode: Some("L1T1".to_owned()), dependency_descriptor: None } + } + } + } +} + +/// Borrowed encoded payload fragment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EncodedFragment<'a> { + /// Encoded fragment bytes. + pub bytes: &'a [u8], +} + +/// Encoded access-unit payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EncodedPayload<'a> { + /// One contiguous payload buffer. + Contiguous(&'a [u8]), + /// Multiple payload fragments. + Fragments(&'a [EncodedFragment<'a>]), + /// Owned payload bytes. + Owned(Vec), +} + +impl EncodedPayload<'_> { + pub(crate) fn is_empty(&self) -> bool { + match self { + Self::Contiguous(bytes) => bytes.is_empty(), + Self::Fragments(fragments) => { + fragments.is_empty() || fragments.iter().any(|fragment| fragment.bytes.is_empty()) + } + Self::Owned(bytes) => bytes.is_empty(), + } + } + + pub(crate) fn to_vec(&self) -> Vec { + match self { + Self::Contiguous(bytes) => bytes.to_vec(), + Self::Fragments(fragments) => { + let len = fragments.iter().map(|fragment| fragment.bytes.len()).sum(); + let mut payload = Vec::with_capacity(len); + for fragment in *fragments { + payload.extend_from_slice(fragment.bytes); + } + payload + } + Self::Owned(bytes) => bytes.clone(), + } + } +} + +/// One encoded video access unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedAccessUnit<'a> { + /// Encoded codec. + pub codec: EncodedVideoCodec, + /// Encoded payload. + pub payload: EncodedPayload<'a>, + /// Capture timestamp in microseconds. + pub timestamp_us: i64, + /// Encoded frame type. + pub frame_type: EncodedFrameType, + /// Encoded frame width in pixels. + pub width: u32, + /// Encoded frame height in pixels. + pub height: u32, + /// Optional layer identifiers. + pub layers: EncodedLayerInfo, + /// Optional codec-specific metadata. + pub codec_specific: CodecSpecific, +} + +/// Owned encoded video access unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OwnedEncodedAccessUnit { + /// Encoded codec. + pub codec: EncodedVideoCodec, + /// Encoded payload bytes. + pub payload: Bytes, + /// Capture timestamp in microseconds. + pub timestamp_us: i64, + /// Encoded frame type. + pub frame_type: EncodedFrameType, + /// Encoded frame width in pixels. + pub width: u32, + /// Encoded frame height in pixels. + pub height: u32, + /// Optional layer identifiers. + pub layers: EncodedLayerInfo, + /// Optional codec-specific metadata. + pub codec_specific: CodecSpecific, +} + +impl OwnedEncodedAccessUnit { + /// Creates an owned encoded access unit from contiguous bytes. + pub fn new( + codec: EncodedVideoCodec, + payload: impl Into, + timestamp_us: i64, + frame_type: EncodedFrameType, + width: u32, + height: u32, + ) -> Self { + Self { + codec, + payload: payload.into(), + timestamp_us, + frame_type, + width, + height, + layers: EncodedLayerInfo::default(), + codec_specific: CodecSpecific::None, + } + } + + /// Borrows this owned access unit as an [`EncodedAccessUnit`]. + pub fn as_access_unit(&self) -> EncodedAccessUnit<'_> { + EncodedAccessUnit { + codec: self.codec, + payload: EncodedPayload::Contiguous(&self.payload), + timestamp_us: self.timestamp_us, + frame_type: self.frame_type, + width: self.width, + height: self.height, + layers: self.layers, + codec_specific: self.codec_specific.clone(), + } + } + + /// Creates an owned access unit by copying a borrowed access unit. + pub fn copy_from(access_unit: &EncodedAccessUnit<'_>) -> Self { + Self { + codec: access_unit.codec, + payload: Bytes::from(access_unit.payload.to_vec()), + timestamp_us: access_unit.timestamp_us, + frame_type: access_unit.frame_type, + width: access_unit.width, + height: access_unit.height, + layers: access_unit.layers, + codec_specific: access_unit.codec_specific.clone(), + } + } +} + +impl<'a> EncodedAccessUnit<'a> { + /// Creates an access unit from one contiguous payload. + pub fn contiguous( + codec: EncodedVideoCodec, + payload: &'a [u8], + timestamp_us: i64, + frame_type: EncodedFrameType, + width: u32, + height: u32, + ) -> Self { + Self { + codec, + payload: EncodedPayload::Contiguous(payload), + timestamp_us, + frame_type, + width, + height, + layers: EncodedLayerInfo::default(), + codec_specific: CodecSpecific::None, + } + } + + /// Creates an H.264 access unit from raw NAL-unit payloads. + pub fn from_h264_nalus( + nal_units: &[&[u8]], + timestamp_us: i64, + width: u32, + height: u32, + ) -> Result, CaptureError> { + Self::from_nalus(EncodedVideoCodec::H264, nal_units, timestamp_us, width, height) + } + + /// Creates an H.265 access unit from raw NAL-unit payloads. + pub fn from_h265_nalus( + nal_units: &[&[u8]], + timestamp_us: i64, + width: u32, + height: u32, + ) -> Result, CaptureError> { + Self::from_nalus(EncodedVideoCodec::H265, nal_units, timestamp_us, width, height) + } + + fn from_nalus( + codec: EncodedVideoCodec, + nal_units: &[&[u8]], + timestamp_us: i64, + width: u32, + height: u32, + ) -> Result, CaptureError> { + let is_key = is_keyframe_nalus(codec, nal_units)?; + Ok(EncodedAccessUnit { + codec, + payload: EncodedPayload::Owned(annex_b_payload(nal_units)?), + timestamp_us, + frame_type: if is_key { EncodedFrameType::Key } else { EncodedFrameType::Delta }, + width, + height, + layers: EncodedLayerInfo::default(), + codec_specific: CodecSpecific::default_for(codec), + }) + } +} + +/// Returns true when the NAL units form a WebRTC-usable key frame. +pub(crate) fn is_keyframe_nalus( + codec: EncodedVideoCodec, + nal_units: &[&[u8]], +) -> Result { + match codec { + EncodedVideoCodec::H264 => { + nal_units.iter().try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == 5)) + } + EncodedVideoCodec::H265 => { + let mut has_vps = false; + let mut has_sps = false; + let mut has_pps = false; + let mut has_idr = false; + + for nal in nal_units { + match h265_nal_type(nal)? { + 32 => has_vps = true, + 33 => has_sps = true, + 34 => has_pps = true, + 19 | 20 => has_idr = true, + _ => {} + } + } + + Ok(has_vps && has_sps && has_pps && has_idr) + } + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + Err(CaptureError::UnsupportedCodec(codec)) + } + } +} + +impl From for VideoCodec { + fn from(value: EncodedVideoCodec) -> Self { + match value { + EncodedVideoCodec::H264 => Self::H264, + EncodedVideoCodec::H265 => Self::H265, + EncodedVideoCodec::VP8 => Self::VP8, + EncodedVideoCodec::VP9 => Self::VP9, + EncodedVideoCodec::AV1 => Self::AV1, + } + } +} + +impl From for RtcEncodedVideoCodec { + fn from(value: EncodedVideoCodec) -> Self { + match value { + EncodedVideoCodec::H264 => Self::H264, + EncodedVideoCodec::H265 => Self::H265, + EncodedVideoCodec::VP8 => Self::VP8, + EncodedVideoCodec::VP9 => Self::VP9, + EncodedVideoCodec::AV1 => Self::AV1, + } + } +} + +impl From for RtcEncodedFrameType { + fn from(value: EncodedFrameType) -> Self { + match value { + EncodedFrameType::Key => Self::Key, + EncodedFrameType::Delta => Self::Delta, + } + } +} + +pub(crate) fn h264_nal_type(nal: &[u8]) -> Result { + let header = nal.first().ok_or(CaptureError::EmptyPayload)?; + Ok(header & 0x1f) +} + +pub(crate) fn h265_nal_type(nal: &[u8]) -> Result { + if nal.is_empty() { + return Err(CaptureError::EmptyPayload); + } + if nal.len() < 2 { + return Err(CaptureError::H265NalTooShort); + } + Ok((nal[0] >> 1) & 0x3f) +} + +pub(crate) fn annex_b_payload(nal_units: &[&[u8]]) -> Result, CaptureError> { + if nal_units.is_empty() { + return Err(CaptureError::EmptyPayload); + } + let len = nal_units.iter().try_fold(0usize, |len, nal| { + if nal.is_empty() { + Err(CaptureError::EmptyPayload) + } else { + Ok(len + ANNEX_B_START_CODE.len() + nal.len()) + } + })?; + + let mut payload = Vec::with_capacity(len); + for nal in nal_units { + payload.extend_from_slice(&ANNEX_B_START_CODE); + payload.extend_from_slice(nal); + } + Ok(payload) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn h264_nal_helper_assembles_annex_b_and_detects_keyframe() { + let sps = [0x67, 1, 2, 3]; + let idr = [0x65, 4, 5, 6]; + let au = EncodedAccessUnit::from_h264_nalus(&[&sps, &idr], 10, 640, 480).unwrap(); + + assert_eq!(au.codec, EncodedVideoCodec::H264); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!( + au.payload, + EncodedPayload::Owned(vec![0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5, 6]) + ); + } + + #[test] + fn h265_nal_helper_requires_parameter_sets_and_idr_keyframe() { + let vps = [0x40, 1, 2]; + let sps = [0x42, 1, 2]; + let pps = [0x44, 1, 2]; + let idr_w_radl = [19 << 1, 1, 3]; + let idr_without_headers = + EncodedAccessUnit::from_h265_nalus(&[&vps, &idr_w_radl], 10, 640, 480).unwrap(); + let key = + EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &idr_w_radl], 10, 640, 480) + .unwrap(); + let cra = [21 << 1, 1, 3]; + let cra_with_headers = + EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &cra], 10, 640, 480).unwrap(); + + assert_eq!(idr_without_headers.codec, EncodedVideoCodec::H265); + assert_eq!(idr_without_headers.frame_type, EncodedFrameType::Delta); + assert_eq!(key.frame_type, EncodedFrameType::Key); + assert_eq!(cra_with_headers.frame_type, EncodedFrameType::Delta); + } + + #[test] + fn h265_rejects_too_short_nal_header() { + let err = EncodedAccessUnit::from_h265_nalus(&[&[0x26]], 10, 640, 480).unwrap_err(); + assert_eq!(err, CaptureError::H265NalTooShort); + } + + #[test] + fn fragments_reject_empty_fragment() { + let fragments = [EncodedFragment { bytes: &[1] }, EncodedFragment { bytes: &[] }]; + let payload = EncodedPayload::Fragments(&fragments); + assert!(payload.is_empty()); + } + + #[test] + fn owned_access_unit_borrows_without_copying_payload() { + let owned = OwnedEncodedAccessUnit::new( + EncodedVideoCodec::H264, + Bytes::from_static(&[1, 2, 3]), + 10, + EncodedFrameType::Delta, + 640, + 480, + ); + + let borrowed = owned.as_access_unit(); + assert_eq!(borrowed.codec, EncodedVideoCodec::H264); + assert_eq!(borrowed.payload, EncodedPayload::Contiguous(&[1, 2, 3])); + assert_eq!(borrowed.timestamp_us, 10); + } +} diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index 344299d24..6b12d8a46 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -14,6 +14,8 @@ use thiserror::Error; +use crate::encoded::{EncodedVideoCodec, EncodedWireFormat}; + /// Error returned by capture helpers. #[derive(Debug, Error, PartialEq, Eq)] pub enum CaptureError { @@ -26,9 +28,15 @@ pub enum CaptureError { /// Access unit carries layering metadata the passthrough cannot forward. #[error("unsupported layered encoding: {0}")] UnsupportedLayeredEncoding(&'static str), + /// Codec is represented by the API but not yet supported by native passthrough. + #[error("encoded passthrough does not support {0:?} yet")] + UnsupportedCodec(EncodedVideoCodec), /// Encoded payload or transport data is malformed. #[error("invalid encoded data: {0}")] InvalidEncodedData(&'static str), + /// Wire format is represented by the API but not supported by this source. + #[error("encoded wire format is not supported by this source: {0:?}")] + UnsupportedWireFormat(EncodedWireFormat), /// Capture backend is not available on this platform. #[error("{0} is not supported on this platform")] UnsupportedPlatform(&'static str), diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 05b2405e0..7896637f7 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -14,6 +14,12 @@ //! Helpers for publishing pre-encoded video with LiveKit. +pub mod encoded; mod error; +pub use encoded::{ + CodecSpecific, EncodedAccessUnit, EncodedFragment, EncodedFrameType, EncodedLayerInfo, + EncodedPayload, EncodedRateControl, EncodedVideoCodec, EncodedWireFormat, + H264PacketizationMode, OwnedEncodedAccessUnit, +}; pub use error::CaptureError; From 6ae2e120439b54a95d489b0777b0fd54125cb6f9 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:42:11 -0700 Subject: [PATCH 04/56] Add h26x access unit parsing --- livekit-capture/src/encoded.rs | 2 + livekit-capture/src/encoded/h26x.rs | 918 ++++++++++++++++++++++++++++ 2 files changed, 920 insertions(+) create mode 100644 livekit-capture/src/encoded/h26x.rs diff --git a/livekit-capture/src/encoded.rs b/livekit-capture/src/encoded.rs index f6b428ab8..1d26e4728 100644 --- a/livekit-capture/src/encoded.rs +++ b/livekit-capture/src/encoded.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod h26x; + use bytes::Bytes; use livekit::{ options::VideoCodec, diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs new file mode 100644 index 000000000..7ee5335d3 --- /dev/null +++ b/livekit-capture/src/encoded/h26x.rs @@ -0,0 +1,918 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::ops::Range; + +use bytes::Bytes; + +use crate::{ + encoded::{ + annex_b_payload, h264_nal_type, h265_nal_type, is_keyframe_nalus, CodecSpecific, + EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, + }, + error::CaptureError, +}; + +/// Upper bound on bytes buffered while waiting for an access-unit boundary. +const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; + +/// Byte-stream access-unit parser shared by the encoded ingest sources. +/// +/// `push` appends bytes and returns at most one completed access unit; call +/// `drain` repeatedly to pull further access units already buffered, and +/// `flush` once at end of stream to emit the final pending access unit. +pub(crate) trait AccessUnitParser { + /// Appends bytes and returns the next complete access unit, if any. + fn push(&mut self, bytes: &[u8]) -> Result, CaptureError>; + + /// Returns the next complete access unit from already-buffered bytes. + fn drain(&mut self) -> Result, CaptureError> { + self.push(&[]) + } + + /// Flushes remaining buffered bytes as the final access unit. + fn flush(&mut self) -> Result, CaptureError>; +} + +/// H26x Annex-B parser state. +#[derive(Debug, Clone)] +pub struct AnnexBAccessUnitParser { + codec: EncodedVideoCodec, + pending: Vec, + /// NAL ranges found in `pending`; the last range's end is provisional + /// until the next start code (or flush) confirms it. + nal_ranges: Vec>, + /// Offset up to which `pending` has been scanned for start codes. + scan_cursor: usize, + next_timestamp_us: i64, + frame_interval_us: i64, + width: u32, + height: u32, +} + +/// H.264/AVC length-prefixed parser state. +#[cfg(any(feature = "tcpsink", test))] +#[derive(Debug, Clone)] +pub(crate) struct AvcAccessUnitParser { + pending: Vec, + /// Complete NAL ranges found in `pending`. + nal_ranges: Vec>, + /// Offset of the first unparsed length prefix or incomplete NAL in `pending`. + scan_cursor: usize, + nal_length_size: u8, + next_timestamp_us: i64, + frame_interval_us: i64, + width: u32, + height: u32, +} + +impl AnnexBAccessUnitParser { + /// Creates a parser for H.264 or H.265 Annex-B byte streams. + pub fn new( + codec: EncodedVideoCodec, + start_timestamp_us: i64, + frame_interval_us: i64, + width: u32, + height: u32, + ) -> Result { + match codec { + EncodedVideoCodec::H264 | EncodedVideoCodec::H265 => {} + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + return Err(CaptureError::UnsupportedCodec(codec)); + } + } + + Ok(Self { + codec, + pending: Vec::new(), + nal_ranges: Vec::new(), + scan_cursor: 0, + next_timestamp_us: start_timestamp_us, + frame_interval_us, + width, + height, + }) + } + + /// Pushes encoded bytes and returns the next complete access unit if one is found. + pub fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { + self.pending.extend_from_slice(bytes); + self.drain_next(false) + } + + /// Flushes the pending bytes as the final access unit. + pub fn flush(&mut self) -> Result, CaptureError> { + self.drain_next(true) + } + + fn drain_next(&mut self, at_eof: bool) -> Result, CaptureError> { + self.scan_pending(); + + if let Some(split_at) = + access_unit_split_index(self.codec, &self.pending, &self.nal_ranges)? + { + return self.take_access_unit(split_at); + } + if at_eof && self.nal_ranges.iter().any(|range| range.start < range.end) { + return self.take_access_unit(self.pending.len()); + } + if !at_eof && self.pending.len() > MAX_PENDING_ACCESS_UNIT_BYTES { + return Err(CaptureError::InvalidEncodedData( + "access unit exceeds maximum buffered size", + )); + } + Ok(None) + } + + /// Scans bytes appended since the previous call, extending the cached NAL ranges. + fn scan_pending(&mut self) { + // Resume behind the previous scan end so a start code straddling the + // boundary is found, but never before the last NAL start so an + // already-found start code is not rediscovered. + let mut cursor = self.scan_cursor.saturating_sub(3); + if let Some(last) = self.nal_ranges.last() { + cursor = cursor.max(last.start); + } + while let Some((offset, prefix_len)) = find_start_code(&self.pending[cursor..]) { + let prefix_start = cursor + offset; + let nal_start = prefix_start + prefix_len; + if let Some(last) = self.nal_ranges.last_mut() { + last.end = prefix_start; + if last.start >= prefix_start { + self.nal_ranges.pop(); + } + } + self.nal_ranges.push(nal_start..nal_start); + cursor = nal_start; + } + if let Some(last) = self.nal_ranges.last_mut() { + last.end = self.pending.len(); + } + self.scan_cursor = self.pending.len(); + } + + fn take_access_unit( + &mut self, + byte_len: usize, + ) -> Result, CaptureError> { + if byte_len == 0 { + return Ok(None); + } + + let access_unit = self.pending[..byte_len].to_vec(); + self.pending.drain(..byte_len); + self.nal_ranges.retain_mut(|range| { + if range.end <= byte_len { + return false; + } + range.start -= byte_len; + range.end -= byte_len; + true + }); + self.scan_cursor -= byte_len; + let timestamp_us = self.next_timestamp_us; + self.next_timestamp_us = self.next_timestamp_us.saturating_add(self.frame_interval_us); + access_unit_from_annex_b( + self.codec, + Bytes::from(access_unit), + timestamp_us, + self.width, + self.height, + ) + .map(Some) + } +} + +impl AccessUnitParser for AnnexBAccessUnitParser { + fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { + AnnexBAccessUnitParser::push(self, bytes) + } + + fn flush(&mut self) -> Result, CaptureError> { + AnnexBAccessUnitParser::flush(self) + } +} + +#[cfg(any(feature = "tcpsink", test))] +impl AvcAccessUnitParser { + /// Creates a parser for H.264/AVC length-prefixed byte streams. + pub(crate) fn new( + nal_length_size: u8, + start_timestamp_us: i64, + frame_interval_us: i64, + width: u32, + height: u32, + ) -> Result { + validate_avc_nal_length_size(nal_length_size)?; + + Ok(Self { + pending: Vec::new(), + nal_ranges: Vec::new(), + scan_cursor: 0, + nal_length_size, + next_timestamp_us: start_timestamp_us, + frame_interval_us, + width, + height, + }) + } + + /// Pushes encoded bytes and returns the next complete access unit if one is found. + pub(crate) fn push( + &mut self, + bytes: &[u8], + ) -> Result, CaptureError> { + self.pending.extend_from_slice(bytes); + self.drain_next(false) + } + + /// Flushes the pending bytes as the final access unit. + pub(crate) fn flush(&mut self) -> Result, CaptureError> { + self.drain_next(true) + } + + fn drain_next(&mut self, at_eof: bool) -> Result, CaptureError> { + self.scan_pending(at_eof)?; + + if let Some(split_at) = avc_access_unit_split_index( + &self.pending, + &self.nal_ranges, + self.nal_length_size as usize, + )? { + return self.take_access_unit(split_at); + } + if at_eof && !self.nal_ranges.is_empty() { + return self.take_access_unit(self.pending.len()); + } + if !at_eof && self.pending.len() > MAX_PENDING_ACCESS_UNIT_BYTES { + return Err(CaptureError::InvalidEncodedData( + "access unit exceeds maximum buffered size", + )); + } + Ok(None) + } + + /// Parses length-prefixed NAL units appended since the previous call. + fn scan_pending(&mut self, at_eof: bool) -> Result<(), CaptureError> { + let nal_length_size = self.nal_length_size as usize; + while self.scan_cursor < self.pending.len() { + if self.pending.len() - self.scan_cursor < nal_length_size { + if at_eof { + return Err(CaptureError::InvalidEncodedData("truncated AVC NAL length")); + } + break; + } + + let nal_start = self.scan_cursor + nal_length_size; + let nal_len = read_avc_nal_length(&self.pending[self.scan_cursor..nal_start]); + if nal_len == 0 { + return Err(CaptureError::InvalidEncodedData("empty AVC NAL unit")); + } + + let Some(nal_end) = nal_start.checked_add(nal_len) else { + return Err(CaptureError::InvalidEncodedData("AVC NAL unit length overflow")); + }; + if nal_end > self.pending.len() { + if at_eof { + return Err(CaptureError::InvalidEncodedData("truncated AVC NAL unit")); + } + break; + } + + self.nal_ranges.push(nal_start..nal_end); + self.scan_cursor = nal_end; + } + Ok(()) + } + + fn take_access_unit( + &mut self, + byte_len: usize, + ) -> Result, CaptureError> { + if byte_len == 0 { + return Ok(None); + } + + let access_unit = self.pending[..byte_len].to_vec(); + self.pending.drain(..byte_len); + self.nal_ranges.retain_mut(|range| { + if range.end <= byte_len { + return false; + } + range.start -= byte_len; + range.end -= byte_len; + true + }); + self.scan_cursor -= byte_len; + let timestamp_us = self.next_timestamp_us; + self.next_timestamp_us = self.next_timestamp_us.saturating_add(self.frame_interval_us); + access_unit_from_h264_avc( + &access_unit, + self.nal_length_size, + timestamp_us, + self.width, + self.height, + ) + .map(Some) + } +} + +#[cfg(any(feature = "tcpsink", test))] +impl AccessUnitParser for AvcAccessUnitParser { + fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { + AvcAccessUnitParser::push(self, bytes) + } + + fn flush(&mut self) -> Result, CaptureError> { + AvcAccessUnitParser::flush(self) + } +} + +/// Returns NAL-unit byte ranges for an Annex-B access unit or stream chunk. +pub fn annex_b_nal_ranges(bytes: &[u8]) -> Vec> { + let mut ranges = Vec::new(); + let mut cursor = 0; + let mut current_start = None; + + while let Some((prefix_start, prefix_len)) = find_start_code(&bytes[cursor..]) { + let prefix_start = cursor + prefix_start; + let nal_start = prefix_start + prefix_len; + if let Some(start) = current_start.replace(nal_start) { + if start < prefix_start { + ranges.push(start..prefix_start); + } + } + cursor = nal_start; + } + + if let Some(start) = current_start { + if start < bytes.len() { + ranges.push(start..bytes.len()); + } + } + + ranges +} + +/// Returns borrowed NAL units from an Annex-B buffer. +pub fn annex_b_nalus(bytes: &[u8]) -> Result, CaptureError> { + let nals = annex_b_nal_ranges(bytes) + .into_iter() + .map(|range| &bytes[range]) + .filter(|nal| !nal.is_empty()) + .collect::>(); + Ok(nals) +} + +/// Creates an Annex-B access unit from H.264/AVC length-prefixed NAL units. +pub(crate) fn access_unit_from_h264_avc( + payload: &[u8], + nal_length_size: u8, + timestamp_us: i64, + width: u32, + height: u32, +) -> Result { + let nals = avc_nalus(payload, nal_length_size)?; + access_unit_from_nalus(EncodedVideoCodec::H264, &nals, timestamp_us, width, height) +} + +/// Creates an access unit from an Annex-B buffer. +pub fn access_unit_from_annex_b( + codec: EncodedVideoCodec, + payload: Bytes, + timestamp_us: i64, + width: u32, + height: u32, +) -> Result { + if payload.is_empty() { + return Err(CaptureError::EmptyPayload); + } + + let frame_type = if is_keyframe_annex_b(codec, &payload)? { + EncodedFrameType::Key + } else { + EncodedFrameType::Delta + }; + let mut access_unit = + OwnedEncodedAccessUnit::new(codec, payload, timestamp_us, frame_type, width, height); + access_unit.codec_specific = CodecSpecific::default_for(codec); + Ok(access_unit) +} + +/// Creates an Annex-B access unit from raw NAL units. +pub fn access_unit_from_nalus( + codec: EncodedVideoCodec, + nal_units: &[&[u8]], + timestamp_us: i64, + width: u32, + height: u32, +) -> Result { + let payload = Bytes::from(annex_b_payload(nal_units)?); + access_unit_from_annex_b(codec, payload, timestamp_us, width, height) +} + +/// Returns true when an Annex-B access unit contains an intra/key picture. +pub fn is_keyframe_annex_b(codec: EncodedVideoCodec, bytes: &[u8]) -> Result { + let nals = annex_b_nalus(bytes)?; + is_keyframe_nalus(codec, &nals) +} + +fn access_unit_split_index( + codec: EncodedVideoCodec, + bytes: &[u8], + ranges: &[Range], +) -> Result, CaptureError> { + match access_unit_boundary_nal(codec, bytes, ranges)? { + Some(index) => split_start_code_index(bytes, ranges[index].start).map(Some), + None => Ok(None), + } +} + +#[cfg(any(feature = "tcpsink", test))] +fn avc_access_unit_split_index( + bytes: &[u8], + ranges: &[Range], + nal_length_size: usize, +) -> Result, CaptureError> { + match access_unit_boundary_nal(EncodedVideoCodec::H264, bytes, ranges)? { + Some(index) => ranges[index] + .start + .checked_sub(nal_length_size) + .ok_or(CaptureError::InvalidEncodedData("missing AVC NAL length")) + .map(Some), + None => Ok(None), + } +} + +/// Returns the index of the first NAL that starts a new access unit, once at +/// least one VCL NAL has been seen in the current one. +fn access_unit_boundary_nal( + codec: EncodedVideoCodec, + bytes: &[u8], + ranges: &[Range], +) -> Result, CaptureError> { + let mut seen_vcl = false; + for (index, range) in ranges.iter().enumerate() { + let nal = &bytes[range.clone()]; + // The final NAL may still be streaming in; wait for its header. + if index + 1 == ranges.len() && nal.len() < min_nal_header_len(codec) { + return Ok(None); + } + if seen_vcl && starts_new_access_unit(codec, nal)? { + return Ok(Some(index)); + } + seen_vcl |= is_vcl_nal(codec, nal)?; + } + Ok(None) +} + +fn min_nal_header_len(codec: EncodedVideoCodec) -> usize { + match codec { + EncodedVideoCodec::H265 => 2, + _ => 1, + } +} + +fn starts_new_access_unit(codec: EncodedVideoCodec, nal: &[u8]) -> Result { + Ok(match codec { + EncodedVideoCodec::H264 => match h264_nal_type(nal)? { + // Prefix SEI(6), SPS(7), PPS(8), and AUD(9) open a new access unit. + 6..=9 => true, + // A VCL NAL opens a new picture when first_mb_in_slice == 0: + // ue(v) == 0 is a lone 1 bit, so the first RBSP bit after the + // header is set. The header byte is nonzero, so the next byte + // cannot be an emulation-prevention byte. + 1..=5 => nal.len() >= 2 && nal[1] & 0x80 != 0, + _ => false, + }, + EncodedVideoCodec::H265 => match h265_nal_type(nal)? { + // VPS(32), SPS(33), PPS(34), AUD(35), and prefix SEI(39). + 32..=35 | 39 => true, + // A VCL NAL opens a new picture when + // first_slice_segment_in_pic_flag (the bit after the 2-byte + // header) is set. nuh_temporal_id_plus1 makes the second header + // byte nonzero, so the next byte cannot be an + // emulation-prevention byte. + 0..=31 => nal.len() >= 3 && nal[2] & 0x80 != 0, + _ => false, + }, + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + return Err(CaptureError::UnsupportedCodec(codec)); + } + }) +} + +fn split_start_code_index(bytes: &[u8], nal_start: usize) -> Result { + if nal_start >= 4 && bytes[nal_start - 4..nal_start] == [0, 0, 0, 1] { + return Ok(nal_start - 4); + } + if nal_start >= 3 && bytes[nal_start - 3..nal_start] == [0, 0, 1] { + return Ok(nal_start - 3); + } + Err(CaptureError::InvalidEncodedData("missing Annex-B start code")) +} + +fn is_vcl_nal(codec: EncodedVideoCodec, nal: &[u8]) -> Result { + Ok(match codec { + EncodedVideoCodec::H264 => (1..=5).contains(&h264_nal_type(nal)?), + EncodedVideoCodec::H265 => h265_nal_type(nal)? <= 31, + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + return Err(CaptureError::UnsupportedCodec(codec)); + } + }) +} + +fn find_start_code(bytes: &[u8]) -> Option<(usize, usize)> { + let mut idx = 0; + while idx + 3 <= bytes.len() { + if bytes[idx..].starts_with(&[0, 0, 1]) { + return Some((idx, 3)); + } + if idx + 4 <= bytes.len() && bytes[idx..].starts_with(&[0, 0, 0, 1]) { + return Some((idx, 4)); + } + idx += 1; + } + None +} + +fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, CaptureError> { + let ranges = avc_nal_ranges(payload, nal_length_size, true)?; + if ranges.is_empty() { + return Err(CaptureError::EmptyPayload); + } + Ok(ranges.into_iter().map(|range| &payload[range]).collect()) +} + +fn avc_nal_ranges( + bytes: &[u8], + nal_length_size: u8, + at_eof: bool, +) -> Result>, CaptureError> { + validate_avc_nal_length_size(nal_length_size)?; + + let nal_length_size = nal_length_size as usize; + let mut ranges = Vec::new(); + let mut cursor = 0; + while cursor < bytes.len() { + if bytes.len() - cursor < nal_length_size { + if at_eof { + return Err(CaptureError::InvalidEncodedData("truncated AVC NAL length")); + } + break; + } + + let nal_len = read_avc_nal_length(&bytes[cursor..cursor + nal_length_size]); + cursor += nal_length_size; + if nal_len == 0 { + return Err(CaptureError::InvalidEncodedData("empty AVC NAL unit")); + } + + let Some(nal_end) = cursor.checked_add(nal_len) else { + return Err(CaptureError::InvalidEncodedData("AVC NAL unit length overflow")); + }; + if nal_end > bytes.len() { + if at_eof { + return Err(CaptureError::InvalidEncodedData("truncated AVC NAL unit")); + } + break; + } + + ranges.push(cursor..nal_end); + cursor = nal_end; + } + + Ok(ranges) +} + +fn read_avc_nal_length(bytes: &[u8]) -> usize { + bytes.iter().fold(0usize, |len, byte| (len << 8) | usize::from(*byte)) +} + +fn validate_avc_nal_length_size(nal_length_size: u8) -> Result<(), CaptureError> { + if (1..=4).contains(&nal_length_size) { + return Ok(()); + } + Err(CaptureError::InvalidEncodedData("invalid AVC NAL length size")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_annex_b_nals_with_three_and_four_byte_prefixes() { + let bytes = [0, 0, 1, 0x67, 1, 0, 0, 0, 1, 0x65, 2, 3]; + let nals = annex_b_nalus(&bytes).unwrap(); + assert_eq!(nals, vec![&[0x67, 1][..], &[0x65, 2, 3][..]]); + } + + #[test] + fn detects_h264_keyframe_from_annex_b() { + let bytes = [0, 0, 0, 1, 0x61, 1, 0, 0, 0, 1, 0x65, 2]; + assert!(is_keyframe_annex_b(EncodedVideoCodec::H264, &bytes).unwrap()); + } + + #[test] + fn access_unit_from_avc_converts_length_prefixed_nals() { + let bytes = [0, 0, 0, 4, 0x67, 1, 2, 3, 0, 0, 0, 3, 0x65, 4, 5]; + let au = access_unit_from_h264_avc(&bytes, 4, 10, 640, 480).unwrap(); + + assert_eq!(au.codec, EncodedVideoCodec::H264); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5]); + } + + #[test] + fn access_unit_from_avc_supports_two_byte_lengths() { + let bytes = [0, 2, 0x61, 1]; + let au = access_unit_from_h264_avc(&bytes, 2, 10, 640, 480).unwrap(); + + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x61, 1]); + } + + #[test] + fn access_unit_from_avc_rejects_truncated_nal() { + let err = access_unit_from_h264_avc(&[0, 0, 0, 3, 0x65], 4, 10, 640, 480).unwrap_err(); + + assert_eq!(err, CaptureError::InvalidEncodedData("truncated AVC NAL unit")); + } + + #[test] + fn parser_flushes_final_access_unit() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, 640, 480).unwrap(); + assert!(parser.push(&[0, 0, 1, 0x65, 1, 2]).unwrap().is_none()); + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 100); + assert_eq!(au.frame_type, EncodedFrameType::Key); + } + + #[test] + fn parser_splits_at_next_access_unit_delimiter() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, 640, 480).unwrap(); + let stream = + [0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x65, 1, 2, 0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x41, 3]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 100); + assert_eq!(au.payload.as_ref(), &[0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x65, 1, 2]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_433); + assert_eq!(au.payload.as_ref(), &[0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x41, 3]); + } + + #[test] + fn avc_parser_splits_at_next_access_unit_delimiter() { + let mut parser = AvcAccessUnitParser::new(4, 100, 33_333, 640, 480).unwrap(); + let stream = [ + 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 3, 0x65, 1, 2, 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 2, + 0x41, 3, + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 100); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x09, 0x10, 0, 0, 0, 1, 0x65, 1, 2]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_433); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x09, 0x10, 0, 0, 0, 1, 0x41, 3]); + } + + #[test] + fn splits_aud_less_h264_stream_per_frame() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + let stream = [ + 0, 0, 0, 1, 0x67, 0x42, 0x00, 0x1e, // SPS + 0, 0, 0, 1, 0x68, 0xce, // PPS + 0, 0, 1, 0x65, 0x88, 0x84, 0x21, // IDR slice, first_mb_in_slice == 0 + 0, 0, 1, 0x41, 0x9a, 0x22, // P slice, first_mb_in_slice == 0 + 0, 0, 1, 0x41, 0x9a, 0x33, // P slice, first_mb_in_slice == 0 + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..21]); + + let au = parser.drain().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &stream[21..27]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 66_666); + assert_eq!(au.payload.as_ref(), &stream[27..]); + } + + #[test] + fn keeps_multi_slice_h264_access_unit_together() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + let stream = [ + 0, 0, 1, 0x65, 0x88, 0x11, // IDR slice, first_mb_in_slice == 0 + 0, 0, 1, 0x65, 0x21, 0x22, // IDR slice, first_mb_in_slice != 0 + 0, 0, 1, 0x41, 0x9a, 0x33, // next picture + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..12]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.payload.as_ref(), &stream[12..]); + } + + #[test] + fn splits_aud_less_h265_stream_per_frame() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, 640, 480).unwrap(); + let stream = [ + 0, 0, 0, 1, 0x40, 0x01, 0x0c, // VPS + 0, 0, 0, 1, 0x42, 0x01, 0x02, // SPS + 0, 0, 0, 1, 0x44, 0x01, 0x03, // PPS + 0, 0, 1, 0x26, 0x01, 0xaf, + 0x04, // IDR_W_RADL, first_slice_segment_in_pic_flag == 1 + 0, 0, 1, 0x02, 0x01, 0xd0, 0x05, // TRAIL_R, first_slice_segment_in_pic_flag == 1 + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..28]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &stream[28..]); + } + + #[test] + fn keeps_multi_slice_h265_access_unit_together() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, 640, 480).unwrap(); + let stream = [ + 0, 0, 1, 0x26, 0x01, 0xaf, + 0x11, // IDR slice, first_slice_segment_in_pic_flag == 1 + 0, 0, 1, 0x26, 0x01, 0x40, + 0x22, // IDR slice, first_slice_segment_in_pic_flag == 0 + 0, 0, 1, 0x02, 0x01, 0xd0, 0x33, // next picture + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &stream[..14]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.payload.as_ref(), &stream[14..]); + } + + #[test] + fn groups_parameter_sets_with_following_frame() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + let stream = [ + 0, 0, 1, 0x67, 0x42, 0x1e, // SPS + 0, 0, 1, 0x68, 0xce, // PPS + 0, 0, 1, 0x65, 0x88, 0x11, // IDR + 0, 0, 1, 0x67, 0x42, 0x1e, // SPS + 0, 0, 1, 0x68, 0xce, // PPS + 0, 0, 1, 0x65, 0x88, 0x22, // IDR + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..17]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[17..]); + } + + fn collect_units( + parser: &mut impl AccessUnitParser, + stream: &[u8], + chunk_size: usize, + ) -> Vec<(Vec, i64, EncodedFrameType)> { + let mut units = Vec::new(); + for chunk in stream.chunks(chunk_size) { + let mut unit = parser.push(chunk).unwrap(); + while let Some(au) = unit { + units.push((au.payload.to_vec(), au.timestamp_us, au.frame_type)); + unit = parser.drain().unwrap(); + } + } + let mut unit = parser.flush().unwrap(); + while let Some(au) = unit { + units.push((au.payload.to_vec(), au.timestamp_us, au.frame_type)); + unit = parser.flush().unwrap(); + } + units + } + + fn assert_chunked_matches_one_shot( + make_parser: impl Fn() -> P, + stream: &[u8], + expected_units: usize, + ) { + let baseline = collect_units(&mut make_parser(), stream, stream.len()); + assert_eq!(baseline.len(), expected_units); + for chunk_size in [1, 7] { + assert_eq!(collect_units(&mut make_parser(), stream, chunk_size), baseline); + } + } + + #[test] + fn chunked_pushes_match_one_shot_parsing() { + let h264_annex_b = [ + 0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1e, // SPS + 0, 0, 0, 1, 0x68, 0xce, 0x3c, 0x80, // PPS + 0, 0, 1, 0x65, 0x88, 0x84, 0x00, 0x01, // IDR, first_mb_in_slice == 0 + 0, 0, 1, 0x41, 0x9a, 0x02, // P, first_mb_in_slice == 0 + 0, 0, 1, 0x09, 0x10, // AUD + 0, 0, 1, 0x41, 0x9a, 0x03, // P + 0, 0, 0, 1, 0x41, 0x9a, 0x04, 0x00, // P, first_mb_in_slice == 0 + ]; + assert_chunked_matches_one_shot( + || AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(), + &h264_annex_b, + 4, + ); + + let h265_annex_b = [ + 0, 0, 0, 1, 0x40, 0x01, 0x0c, // VPS + 0, 0, 0, 1, 0x42, 0x01, 0x02, // SPS + 0, 0, 0, 1, 0x44, 0x01, 0x03, // PPS + 0, 0, 1, 0x26, 0x01, 0xaf, 0x08, // IDR_W_RADL + 0, 0, 1, 0x02, 0x01, 0xd0, 0x09, // TRAIL_R + 0, 0, 1, 0x46, 0x01, 0x50, // AUD + 0, 0, 1, 0x02, 0x01, 0xd0, 0x0a, // TRAIL_R + ]; + assert_chunked_matches_one_shot( + || AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, 640, 480).unwrap(), + &h265_annex_b, + 3, + ); + + let h264_avc = [ + 0, 0, 0, 4, 0x67, 0x64, 0x00, 0x1e, // SPS + 0, 0, 0, 2, 0x68, 0xce, // PPS + 0, 0, 0, 4, 0x65, 0x88, 0x84, 0x00, // IDR, first_mb_in_slice == 0 + 0, 0, 0, 3, 0x41, 0x9a, 0x02, // P, first_mb_in_slice == 0 + 0, 0, 0, 2, 0x09, 0x10, // AUD + 0, 0, 0, 3, 0x41, 0x9a, 0x03, // P + ]; + assert_chunked_matches_one_shot( + || AvcAccessUnitParser::new(4, 0, 33_333, 640, 480).unwrap(), + &h264_avc, + 3, + ); + } + + #[test] + fn rejects_pending_access_unit_over_size_cap() { + let mut parser = + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + assert!(parser.push(&[0, 0, 1, 0x65, 0x88]).unwrap().is_none()); + + let err = parser.push(&vec![0xff; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); + assert_eq!( + err, + CaptureError::InvalidEncodedData("access unit exceeds maximum buffered size") + ); + } + + #[test] + fn avc_rejects_pending_access_unit_over_size_cap() { + let mut parser = AvcAccessUnitParser::new(4, 0, 33_333, 640, 480).unwrap(); + let nal_len = (MAX_PENDING_ACCESS_UNIT_BYTES + 1) as u32; + assert!(parser.push(&nal_len.to_be_bytes()).unwrap().is_none()); + + let err = parser.push(&vec![0x41; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); + assert_eq!( + err, + CaptureError::InvalidEncodedData("access unit exceeds maximum buffered size") + ); + } +} From 2c0d2c3829e888af67f301122d63b21fa9cc5e9e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:42:56 -0700 Subject: [PATCH 05/56] Add video capture track --- livekit-capture/src/lib.rs | 2 + livekit-capture/src/track.rs | 249 +++++++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 livekit-capture/src/track.rs diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 7896637f7..535d53c54 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -16,6 +16,7 @@ pub mod encoded; mod error; +pub mod track; pub use encoded::{ CodecSpecific, EncodedAccessUnit, EncodedFragment, EncodedFrameType, EncodedLayerInfo, @@ -23,3 +24,4 @@ pub use encoded::{ H264PacketizationMode, OwnedEncodedAccessUnit, }; pub use error::CaptureError; +pub use track::VideoCaptureTrack; diff --git a/livekit-capture/src/track.rs b/livekit-capture/src/track.rs new file mode 100644 index 000000000..fb825e335 --- /dev/null +++ b/livekit-capture/src/track.rs @@ -0,0 +1,249 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use livekit::{ + options::{TrackPublishOptions, VideoEncoderBackend}, + prelude::LocalVideoTrack, + webrtc::{ + video_frame::{EncodedVideoFrame, FrameMetadata, VideoBuffer, VideoFrame}, + video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution}, + }, +}; + +use crate::{ + encoded::{ + CodecSpecific, EncodedAccessUnit, EncodedLayerInfo, EncodedPayload, EncodedRateControl, + EncodedVideoCodec, + }, + error::CaptureError, +}; + +/// Capture source backed by a LiveKit local video track. +#[derive(Debug, Clone)] +pub struct VideoCaptureTrack { + source: NativeVideoSource, + track: LocalVideoTrack, +} + +impl VideoCaptureTrack { + /// Creates a capture track with the supplied resolution. + pub fn new(name: &str, resolution: VideoResolution, is_screencast: bool) -> Self { + let source = NativeVideoSource::new(resolution, is_screencast); + let track = + LocalVideoTrack::create_video_track(name, RtcVideoSource::Native(source.clone())); + Self { source, track } + } + + /// Creates a capture track for pre-encoded access units. + /// + /// Unlike [`VideoCaptureTrack::new`], no raw keepalive frames are + /// injected before the first capture, so the sender starts directly on + /// the passthrough encoder instead of briefly encoding black frames. + pub fn new_encoded(name: &str, resolution: VideoResolution) -> Self { + let source = NativeVideoSource::new_encoded(resolution); + let track = + LocalVideoTrack::create_video_track(name, RtcVideoSource::Native(source.clone())); + Self { source, track } + } + + /// Returns the publishable local video track. + pub fn track(&self) -> LocalVideoTrack { + self.track.clone() + } + + /// Captures one decoded video frame. + pub fn capture_frame>(&self, frame: &VideoFrame) { + self.source.capture_frame(frame); + } + + /// Captures one encoded video access unit. + /// + /// The passthrough path forwards single-layer streams: access units + /// carrying temporal/spatial layer ids, an AV1 dependency descriptor, or + /// a non-`L1T1` scalability mode are rejected so callers are not misled + /// into thinking that metadata reaches the wire. + pub fn capture_encoded(&self, access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError> { + self.capture_encoded_with_metadata(access_unit, None) + } + + /// Captures one encoded video access unit with optional frame metadata. + /// + /// Metadata is only propagated to subscribers when the corresponding + /// [`TrackPublishOptions::frame_metadata_features`] are enabled before + /// publishing the local track. + pub fn capture_encoded_with_metadata( + &self, + access_unit: &EncodedAccessUnit<'_>, + frame_metadata: Option, + ) -> Result<(), CaptureError> { + validate_encoded_access_unit(access_unit)?; + + let scratch; + let payload: &[u8] = match &access_unit.payload { + EncodedPayload::Contiguous(bytes) => bytes, + EncodedPayload::Owned(bytes) => bytes, + EncodedPayload::Fragments(_) => { + scratch = access_unit.payload.to_vec(); + &scratch + } + }; + let frame = EncodedVideoFrame { + codec: access_unit.codec.into(), + payload, + timestamp_us: access_unit.timestamp_us, + frame_type: access_unit.frame_type.into(), + resolution: VideoResolution { width: access_unit.width, height: access_unit.height }, + frame_metadata, + }; + self.source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) + } + + /// Returns and clears the pending keyframe request raised by the + /// passthrough encoder (PLI/FIR from the SFU, late subscriber join, or + /// sender reconfiguration). + /// + /// Poll this from the capture loop and forward the request to the + /// upstream encoder so it produces an IDR; until one arrives, new + /// subscribers cannot render the track. + pub fn take_keyframe_request(&self) -> bool { + self.source.take_keyframe_request() + } + + /// Returns and clears the pending rate-control target raised by the + /// passthrough encoder. + /// + /// Poll this from the capture loop and forward the target to the + /// upstream encoder so congestion control can adjust the produced + /// bitrate. + pub fn take_rate_control_request(&self) -> Option { + self.source.take_rate_control_request() + } + + /// Returns publish options appropriate for encoded passthrough. + pub fn encoded_publish_options(codec: EncodedVideoCodec) -> TrackPublishOptions { + TrackPublishOptions { + video_codec: codec.into(), + video_encoder: VideoEncoderBackend::PreEncoded, + simulcast: false, + ..Default::default() + } + } +} + +fn validate_encoded_access_unit(access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError> { + if access_unit.payload.is_empty() { + return Err(CaptureError::EmptyPayload); + } + if access_unit.layers != EncodedLayerInfo::default() { + return Err(CaptureError::UnsupportedLayeredEncoding( + "temporal/spatial layer ids are not forwarded by the passthrough encoder", + )); + } + let default_specific = CodecSpecific::default_for(access_unit.codec); + if access_unit.codec_specific != CodecSpecific::None + && access_unit.codec_specific != default_specific + { + return Err(CaptureError::UnsupportedLayeredEncoding( + "codec-specific layering metadata is not forwarded by the passthrough encoder", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::encoded::EncodedFrameType; + + #[test] + fn accepts_vp8_vp9_and_av1_access_units() { + for codec in [EncodedVideoCodec::VP8, EncodedVideoCodec::VP9, EncodedVideoCodec::AV1] { + let access_unit = EncodedAccessUnit::contiguous( + codec, + &[1, 2, 3], + 0, + EncodedFrameType::Key, + 640, + 480, + ); + + assert!(validate_encoded_access_unit(&access_unit).is_ok()); + } + } + + #[test] + fn rejects_empty_encoded_access_units() { + let access_unit = EncodedAccessUnit::contiguous( + EncodedVideoCodec::VP8, + &[], + 0, + EncodedFrameType::Key, + 640, + 480, + ); + + assert_eq!(validate_encoded_access_unit(&access_unit), Err(CaptureError::EmptyPayload)); + } + + #[test] + fn accepts_default_codec_specific_metadata() { + let mut access_unit = EncodedAccessUnit::contiguous( + EncodedVideoCodec::AV1, + &[1, 2, 3], + 0, + EncodedFrameType::Key, + 640, + 480, + ); + access_unit.codec_specific = CodecSpecific::default_for(EncodedVideoCodec::AV1); + + assert!(validate_encoded_access_unit(&access_unit).is_ok()); + } + + #[test] + fn rejects_layered_access_units() { + let mut access_unit = EncodedAccessUnit::contiguous( + EncodedVideoCodec::VP9, + &[1, 2, 3], + 0, + EncodedFrameType::Key, + 640, + 480, + ); + access_unit.layers = EncodedLayerInfo { spatial_id: None, temporal_id: Some(1) }; + + assert!(matches!( + validate_encoded_access_unit(&access_unit), + Err(CaptureError::UnsupportedLayeredEncoding(_)) + )); + } + + #[test] + fn rejects_non_default_codec_specific_metadata() { + let mut access_unit = EncodedAccessUnit::contiguous( + EncodedVideoCodec::VP8, + &[1, 2, 3], + 0, + EncodedFrameType::Key, + 640, + 480, + ); + access_unit.codec_specific = CodecSpecific::VP8 { temporal_id: Some(1), layer_sync: true }; + + assert!(matches!( + validate_encoded_access_unit(&access_unit), + Err(CaptureError::UnsupportedLayeredEncoding(_)) + )); + } +} From e7a8b0e25492e1b27ad527493d7931b9d2a47922 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:46:42 -0700 Subject: [PATCH 06/56] Add encoded ingress --- livekit-capture/src/encoded.rs | 1 + livekit-capture/src/encoded/ingress.rs | 314 +++++++++++++++++++++++++ livekit-capture/src/lib.rs | 4 + 3 files changed, 319 insertions(+) create mode 100644 livekit-capture/src/encoded/ingress.rs diff --git a/livekit-capture/src/encoded.rs b/livekit-capture/src/encoded.rs index 1d26e4728..ed889c6cf 100644 --- a/livekit-capture/src/encoded.rs +++ b/livekit-capture/src/encoded.rs @@ -13,6 +13,7 @@ // limitations under the License. pub mod h26x; +pub mod ingress; use bytes::Bytes; use livekit::{ diff --git a/livekit-capture/src/encoded/ingress.rs b/livekit-capture/src/encoded/ingress.rs new file mode 100644 index 000000000..de1864e5a --- /dev/null +++ b/livekit-capture/src/encoded/ingress.rs @@ -0,0 +1,314 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + error::Error, + fmt, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +use livekit::webrtc::video_frame::FrameMetadata; + +use crate::{ + encoded::{EncodedFrameType, EncodedRateControl, OwnedEncodedAccessUnit}, + error::CaptureError, + track::VideoCaptureTrack, +}; + +/// Source of owned encoded access units. +pub trait EncodedAccessUnitSource { + /// Error returned by the source. + type Error: Error + Send + Sync + 'static; + + /// Returns the next encoded access unit, or `Ok(None)` when the source reaches EOF. + fn next_access_unit(&mut self) -> Result, Self::Error>; + + /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to + /// the producer so it can emit an IDR. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn request_keyframe(&mut self) {} + + /// Forwards a downstream rate-control target to the producer. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn update_rate_control(&mut self, _rate_control: EncodedRateControl) {} +} + +/// Error returned while forwarding encoded access units into a track. +#[derive(Debug)] +pub enum EncodedIngressError { + /// The encoded source failed. + Source(E), + /// The capture track rejected an access unit. + Capture(CaptureError), +} + +impl fmt::Display for EncodedIngressError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Source(err) => write!(f, "encoded source failed: {err}"), + Self::Capture(err) => write!(f, "encoded capture failed: {err}"), + } + } +} + +impl Error for EncodedIngressError +where + E: Error + 'static, +{ + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Source(err) => Some(err), + Self::Capture(err) => Some(err), + } + } +} + +/// Cancellation handle for [`EncodedIngress::run_until_end`]. +/// +/// Cheap to clone; wire it to a shutdown signal (e.g. Ctrl-C) and call +/// [`EncodedIngressStop::stop`] from any thread to make the ingest loop +/// return after the access unit in flight. +#[derive(Debug, Clone, Default)] +pub struct EncodedIngressStop(Arc); + +impl EncodedIngressStop { + /// Creates an un-stopped handle. + pub fn new() -> Self { + Self::default() + } + + /// Signals the ingest loop to stop. + pub fn stop(&self) { + self.0.store(true, Ordering::Release); + } + + /// Returns true once [`EncodedIngressStop::stop`] has been called. + pub fn is_stopped(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + +/// Pulls encoded access units from a source and forwards them into a video track. +#[derive(Debug)] +pub struct EncodedIngress { + track: VideoCaptureTrack, + source: S, + stop: EncodedIngressStop, + awaiting_initial_keyframe: bool, +} + +impl EncodedIngress { + /// Creates an encoded ingress runner. + pub fn new(track: VideoCaptureTrack, source: S) -> Self { + Self { track, source, stop: EncodedIngressStop::new(), awaiting_initial_keyframe: true } + } + + /// Returns a cancellation handle for this runner. + pub fn stop_handle(&self) -> EncodedIngressStop { + self.stop.clone() + } + + /// Returns the capture track used by this runner. + pub fn track(&self) -> &VideoCaptureTrack { + &self.track + } + + /// Returns the underlying encoded source. + pub fn source(&self) -> &S { + &self.source + } + + /// Returns the underlying encoded source mutably. + pub fn source_mut(&mut self) -> &mut S { + &mut self.source + } + + /// Consumes this runner and returns its parts. + pub fn into_parts(self) -> (VideoCaptureTrack, S) { + (self.track, self.source) + } +} + +/// Details of one access unit captured by [`EncodedIngress::capture_next`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EncodedIngressCapture { + /// Capture timestamp of the access unit in microseconds. + pub timestamp_us: i64, + /// Frame type of the access unit. + pub frame_type: crate::encoded::EncodedFrameType, + /// Payload size in bytes. + pub payload_len: usize, +} + +impl EncodedIngress +where + S: EncodedAccessUnitSource, +{ + /// Captures the next access unit, returning `None` after source EOF. + /// + /// Downstream rate-control and keyframe requests raised by the + /// passthrough encoder are polled on every call and forwarded to the + /// source via [`EncodedAccessUnitSource::update_rate_control`] and + /// [`EncodedAccessUnitSource::request_keyframe`]. + pub fn capture_next( + &mut self, + ) -> Result, EncodedIngressError> { + self.capture_next_with_metadata(|_| None) + } + + /// Captures the next access unit with metadata generated after the source yields it. + /// + /// The metadata producer is not called for skipped pre-roll frames while + /// the ingress runner is waiting for the initial keyframe. + pub fn capture_next_with_metadata( + &mut self, + frame_metadata: impl FnOnce(&OwnedEncodedAccessUnit) -> Option, + ) -> Result, EncodedIngressError> { + if let Some(rate_control) = self.track.take_rate_control_request() { + self.source.update_rate_control(rate_control); + } + if self.track.take_keyframe_request() { + self.source.request_keyframe(); + } + + let access_unit = loop { + let Some(access_unit) = + self.source.next_access_unit().map_err(EncodedIngressError::Source)? + else { + return Ok(None); + }; + + if !self.awaiting_initial_keyframe || access_unit.frame_type == EncodedFrameType::Key { + self.awaiting_initial_keyframe = false; + break access_unit; + } + }; + + let frame_metadata = frame_metadata(&access_unit); + self.track + .capture_encoded_with_metadata(&access_unit.as_access_unit(), frame_metadata) + .map_err(EncodedIngressError::Capture)?; + Ok(Some(EncodedIngressCapture { + timestamp_us: access_unit.timestamp_us, + frame_type: access_unit.frame_type, + payload_len: access_unit.payload.len(), + })) + } + + /// Captures access units until the source reaches EOF or the stop + /// handle fires, returning the number of captured access units. + pub fn run_until_end(&mut self) -> Result> { + let mut captured = 0; + while !self.stop.is_stopped() && self.capture_next()?.is_some() { + captured += 1; + } + Ok(captured) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::VecDeque, error::Error, fmt}; + + use livekit::webrtc::video_source::VideoResolution; + + use super::*; + use crate::encoded::EncodedVideoCodec; + + #[derive(Debug)] + struct FakeSourceError; + + impl fmt::Display for FakeSourceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("fake source failed") + } + } + + impl Error for FakeSourceError {} + + #[derive(Debug)] + struct FakeSource { + access_units: VecDeque, + } + + impl FakeSource { + fn new(access_units: impl IntoIterator) -> Self { + Self { access_units: access_units.into_iter().collect() } + } + } + + impl EncodedAccessUnitSource for FakeSource { + type Error = FakeSourceError; + + fn next_access_unit(&mut self) -> Result, Self::Error> { + Ok(self.access_units.pop_front()) + } + } + + fn access_unit(timestamp_us: i64, frame_type: EncodedFrameType) -> OwnedEncodedAccessUnit { + OwnedEncodedAccessUnit::new( + EncodedVideoCodec::VP8, + vec![1, 2, 3], + timestamp_us, + frame_type, + 640, + 480, + ) + } + + fn encoded_track() -> VideoCaptureTrack { + VideoCaptureTrack::new_encoded("test", VideoResolution { width: 640, height: 480 }) + } + + #[test] + fn capture_next_starts_at_initial_keyframe() { + let source = FakeSource::new([ + access_unit(1, EncodedFrameType::Delta), + access_unit(2, EncodedFrameType::Delta), + access_unit(3, EncodedFrameType::Key), + ]); + let mut ingress = EncodedIngress::new(encoded_track(), source); + + let capture = ingress + .capture_next() + .expect("capture should succeed") + .expect("keyframe should be captured"); + + assert_eq!(capture.timestamp_us, 3); + assert_eq!(capture.frame_type, EncodedFrameType::Key); + } + + #[test] + fn capture_next_allows_deltas_after_initial_keyframe() { + let source = FakeSource::new([ + access_unit(1, EncodedFrameType::Key), + access_unit(2, EncodedFrameType::Delta), + ]); + let mut ingress = EncodedIngress::new(encoded_track(), source); + + let first = ingress.capture_next().unwrap().unwrap(); + let second = ingress.capture_next().unwrap().unwrap(); + + assert_eq!(first.frame_type, EncodedFrameType::Key); + assert_eq!(second.frame_type, EncodedFrameType::Delta); + assert_eq!(second.timestamp_us, 2); + } +} diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 535d53c54..f8cd8caa1 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -19,6 +19,10 @@ mod error; pub mod track; pub use encoded::{ + ingress::{ + EncodedAccessUnitSource, EncodedIngress, EncodedIngressCapture, EncodedIngressError, + EncodedIngressStop, + }, CodecSpecific, EncodedAccessUnit, EncodedFragment, EncodedFrameType, EncodedLayerInfo, EncodedPayload, EncodedRateControl, EncodedVideoCodec, EncodedWireFormat, H264PacketizationMode, OwnedEncodedAccessUnit, From 4f043752f15d8e4d59bc27dce89a1b4b73b5c34e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:48:13 -0700 Subject: [PATCH 07/56] Add gstreamer source --- livekit-capture/Cargo.toml | 6 + livekit-capture/src/lib.rs | 1 + livekit-capture/src/sources/gstreamer.rs | 831 +++++++++++++++++++++++ livekit-capture/src/sources/mod.rs | 18 + 4 files changed, 856 insertions(+) create mode 100644 livekit-capture/src/sources/gstreamer.rs create mode 100644 livekit-capture/src/sources/mod.rs diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index edf9f9991..d82996f09 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -9,6 +9,12 @@ repository.workspace = true [dependencies] bytes = { workspace = true } +gstreamer = { version = "0.25.2", optional = true } +gstreamer-app = { version = "0.25.2", optional = true } livekit = { workspace = true } log = { workspace = true } thiserror = { workspace = true } + +[features] +default = [] +gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index f8cd8caa1..bbc3b339d 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -16,6 +16,7 @@ pub mod encoded; mod error; +pub mod sources; pub mod track; pub use encoded::{ diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs new file mode 100644 index 000000000..3f5f93296 --- /dev/null +++ b/livekit-capture/src/sources/gstreamer.rs @@ -0,0 +1,831 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::error::Error as StdError; + +use bytes::Bytes; +use thiserror::Error; + +use ::gstreamer as gst; +use ::gstreamer_app as gst_app; +use gst::glib; +use gst::prelude::*; + +use crate::{ + encoded::{ + h26x::{access_unit_from_annex_b, access_unit_from_h264_avc}, + ingress::EncodedAccessUnitSource, + CodecSpecific, EncodedFrameType, EncodedRateControl, EncodedVideoCodec, + OwnedEncodedAccessUnit, + }, + error::CaptureError, +}; + +/// Encoded sample format expected from a GStreamer appsink. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum GStreamerSampleFormat { + /// H.264 Annex-B access units, usually from `h264parse` with byte-stream caps. + H264AnnexB, + /// H.264 access units with AVC length-prefixed NAL units. + H264Avc { + /// Length-prefix size in bytes. + nal_length_size: u8, + }, + /// H.265 Annex-B access units, usually from `h265parse` with byte-stream caps. + H265AnnexB, + /// One already-delimited encoded access unit per appsink sample. + AccessUnit { + /// Codec carried by each appsink sample. + codec: EncodedVideoCodec, + }, +} + +impl GStreamerSampleFormat { + /// Returns the encoded codec carried by this sample format. + pub fn codec(self) -> EncodedVideoCodec { + match self { + Self::H264AnnexB => EncodedVideoCodec::H264, + Self::H264Avc { .. } => EncodedVideoCodec::H264, + Self::H265AnnexB => EncodedVideoCodec::H265, + Self::AccessUnit { codec } => codec, + } + } +} + +/// Configuration for a GStreamer appsink encoded source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GStreamerAppSinkConfig { + /// Format of encoded buffers pulled from appsink. + pub sample_format: GStreamerSampleFormat, + /// Timestamp added to the first buffer timestamp, or used directly as fallback. + pub start_timestamp_us: i64, + /// Fallback frame interval when a GStreamer buffer has no PTS or DTS. + pub frame_interval_us: i64, + /// Encoded frame width in pixels. + pub width: u32, + /// Encoded frame height in pixels. + pub height: u32, +} + +impl GStreamerAppSinkConfig { + /// Creates GStreamer appsink source configuration. + pub fn new( + sample_format: GStreamerSampleFormat, + start_timestamp_us: i64, + frame_interval_us: i64, + width: u32, + height: u32, + ) -> Self { + Self { sample_format, start_timestamp_us, frame_interval_us, width, height } + } +} + +/// Bitrate unit used by a GStreamer encoder property. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GStreamerBitrateUnit { + /// The encoder property expects bits per second. + BitsPerSecond, + /// The encoder property expects kilobits per second. + KilobitsPerSecond, +} + +impl GStreamerBitrateUnit { + fn property_value(self, target_bitrate_bps: u64) -> u64 { + match self { + Self::BitsPerSecond => target_bitrate_bps, + Self::KilobitsPerSecond => target_bitrate_bps.saturating_add(999) / 1000, + } + } +} + +/// GStreamer encoder bitrate control used by [`GStreamerAppSinkEncodedSource`]. +#[derive(Debug, Clone)] +pub struct GStreamerEncoderRateControl { + encoder: gst::Element, + bitrate_property: String, + bitrate_unit: GStreamerBitrateUnit, + last_target_bitrate_bps: Option, +} + +impl GStreamerEncoderRateControl { + /// Creates bitrate control for a GStreamer encoder element. + pub fn new( + encoder: gst::Element, + bitrate_property: &str, + bitrate_unit: GStreamerBitrateUnit, + ) -> Self { + Self { + encoder, + bitrate_property: bitrate_property.to_owned(), + bitrate_unit, + last_target_bitrate_bps: None, + } + } + + fn update(&mut self, rate_control: EncodedRateControl) { + if self.last_target_bitrate_bps == Some(rate_control.target_bitrate_bps) { + return; + } + + let property_value = self.bitrate_unit.property_value(rate_control.target_bitrate_bps); + if set_integer_property(&self.encoder, &self.bitrate_property, property_value) { + self.last_target_bitrate_bps = Some(rate_control.target_bitrate_bps); + log::debug!( + "updated GStreamer encoder '{}' {}={} for WebRTC target {} bps at {:.2} fps", + self.encoder.name(), + self.bitrate_property, + property_value, + rate_control.target_bitrate_bps, + rate_control.framerate_fps, + ); + } + } +} + +/// Encoded source backed by a GStreamer appsink. +#[derive(Debug)] +pub struct GStreamerAppSinkEncodedSource { + appsink: gst_app::AppSink, + config: GStreamerAppSinkConfig, + next_fallback_timestamp_us: i64, + rate_control: Option, +} + +impl GStreamerAppSinkEncodedSource { + /// Creates an encoded source from an existing GStreamer appsink. + pub fn new(appsink: gst_app::AppSink, config: GStreamerAppSinkConfig) -> Self { + Self { + appsink, + config, + next_fallback_timestamp_us: config.start_timestamp_us, + rate_control: None, + } + } + + /// Sets the encoder bitrate control used for downstream rate requests. + pub fn set_encoder_rate_control(&mut self, rate_control: GStreamerEncoderRateControl) { + self.rate_control = Some(rate_control); + } + + /// Returns the wrapped appsink. + pub fn appsink(&self) -> &gst_app::AppSink { + &self.appsink + } + + /// Returns the source configuration. + pub fn config(&self) -> GStreamerAppSinkConfig { + self.config + } + + /// Consumes this source and returns the wrapped appsink. + pub fn into_appsink(self) -> gst_app::AppSink { + self.appsink + } + + fn access_unit_from_sample( + &mut self, + sample: &gst::Sample, + ) -> Result { + let buffer = sample.buffer().ok_or(GStreamerSourceError::MissingBuffer)?; + let timestamp_us = self.timestamp_us(buffer); + let frame_type = if buffer.flags().contains(gst::BufferFlags::DELTA_UNIT) { + EncodedFrameType::Delta + } else { + EncodedFrameType::Key + }; + + let map = buffer + .map_readable() + .map_err(|err| GStreamerSourceError::MapReadable(err.to_string()))?; + let payload = map.as_ref(); + access_unit_from_sample_payload( + self.config.sample_format, + payload, + timestamp_us, + frame_type, + self.config.width, + self.config.height, + ) + .map_err(GStreamerSourceError::Capture) + } + + fn timestamp_us(&mut self, buffer: &gst::BufferRef) -> i64 { + if let Some(timestamp) = buffer.pts().or_else(|| buffer.dts()) { + let timestamp_us = + clock_time_to_timestamp_us(self.config.start_timestamp_us, timestamp); + self.next_fallback_timestamp_us = + timestamp_us.saturating_add(self.config.frame_interval_us); + return timestamp_us; + } + + let timestamp_us = self.next_fallback_timestamp_us; + self.next_fallback_timestamp_us = + self.next_fallback_timestamp_us.saturating_add(self.config.frame_interval_us); + timestamp_us + } +} + +impl EncodedAccessUnitSource for GStreamerAppSinkEncodedSource { + type Error = GStreamerSourceError; + + fn next_access_unit(&mut self) -> Result, Self::Error> { + match self.appsink.pull_sample() { + Ok(sample) => self.access_unit_from_sample(&sample).map(Some), + Err(_err) if self.appsink.is_eos() => Ok(None), + Err(err) => Err(GStreamerSourceError::PullSample(err.to_string())), + } + } + + fn request_keyframe(&mut self) { + // The `GstForceKeyUnit` custom upstream event is understood by every + // GStreamer video encoder (it is what gst-video's force-key-unit + // helper builds), so downstream PLI/FIR reaches the producer. + let structure = + gst::Structure::builder("GstForceKeyUnit").field("all-headers", true).build(); + let _ = self.appsink.send_event(gst::event::CustomUpstream::new(structure)); + } + + fn update_rate_control(&mut self, rate_control: EncodedRateControl) { + if let Some(control) = &mut self.rate_control { + control.update(rate_control); + } + } +} + +fn set_integer_property(element: &gst::Element, property: &str, value: u64) -> bool { + let Some(pspec) = element.find_property(property) else { + log::warn!("GStreamer encoder '{}' has no '{property}' property", element.name()); + return false; + }; + + let flags = pspec.flags(); + if !flags.contains(glib::ParamFlags::WRITABLE) + || flags.contains(glib::ParamFlags::CONSTRUCT_ONLY) + { + log::warn!("GStreamer encoder '{}' property '{property}' is not writable", element.name()); + return false; + } + + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property( + property, + value.clamp(pspec.minimum() as u64, pspec.maximum() as u64) as u32, + ); + return true; + } + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property( + property, + clamp_to_i64(value, pspec.minimum() as i64, pspec.maximum() as i64) as i32, + ); + return true; + } + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property(property, value.clamp(pspec.minimum(), pspec.maximum())); + return true; + } + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property(property, clamp_to_i64(value, pspec.minimum(), pspec.maximum())); + return true; + } + + log::warn!( + "GStreamer encoder '{}' property '{property}' has unsupported type '{}'", + element.name(), + pspec.value_type() + ); + false +} + +fn clamp_to_i64(value: u64, minimum: i64, maximum: i64) -> i64 { + let value = value.min(i64::MAX as u64) as i64; + value.clamp(minimum, maximum) +} + +/// Error returned by GStreamer appsink encoded sources. +#[derive(Debug, Error)] +pub enum GStreamerSourceError { + /// The appsink failed to produce a sample. + #[error("failed to pull GStreamer appsink sample: {0}")] + PullSample(String), + /// The sample did not contain an encoded buffer. + #[error("GStreamer sample did not contain a buffer")] + MissingBuffer, + /// The sample buffer could not be mapped for reading. + #[error("failed to map GStreamer buffer for reading: {0}")] + MapReadable(String), + /// Access-unit construction failed. + #[error(transparent)] + Capture(CaptureError), +} + +/// Callback-backed encoded source for GStreamer appsink integrations. +#[derive(Debug)] +pub struct GStreamerAppSinkSource { + next_access_unit: F, +} + +impl GStreamerAppSinkSource { + /// Creates a source from a callback that pulls the next encoded appsink sample. + pub fn new(next_access_unit: F) -> Self { + Self { next_access_unit } + } + + /// Returns the wrapped callback. + pub fn callback(&self) -> &F { + &self.next_access_unit + } + + /// Returns the wrapped callback mutably. + pub fn callback_mut(&mut self) -> &mut F { + &mut self.next_access_unit + } + + /// Consumes this source and returns the wrapped callback. + pub fn into_callback(self) -> F { + self.next_access_unit + } +} + +impl EncodedAccessUnitSource for GStreamerAppSinkSource +where + F: FnMut() -> Result, E>, + E: StdError + Send + Sync + 'static, +{ + type Error = E; + + fn next_access_unit(&mut self) -> Result, Self::Error> { + (self.next_access_unit)() + } +} + +fn access_unit_from_sample_payload( + sample_format: GStreamerSampleFormat, + payload: &[u8], + timestamp_us: i64, + frame_type: EncodedFrameType, + width: u32, + height: u32, +) -> Result { + match sample_format { + GStreamerSampleFormat::H264AnnexB => access_unit_from_annex_b( + EncodedVideoCodec::H264, + Bytes::copy_from_slice(payload), + timestamp_us, + width, + height, + ), + GStreamerSampleFormat::H264Avc { nal_length_size } => { + access_unit_from_h264_avc(payload, nal_length_size, timestamp_us, width, height) + } + GStreamerSampleFormat::H265AnnexB => access_unit_from_annex_b( + EncodedVideoCodec::H265, + Bytes::copy_from_slice(payload), + timestamp_us, + width, + height, + ), + GStreamerSampleFormat::AccessUnit { codec } => { + if payload.is_empty() { + return Err(CaptureError::EmptyPayload); + } + + let mut access_unit = OwnedEncodedAccessUnit::new( + codec, + Bytes::copy_from_slice(payload), + timestamp_us, + frame_type, + width, + height, + ); + access_unit.codec_specific = CodecSpecific::default_for(codec); + Ok(access_unit) + } + } +} + +fn clock_time_to_timestamp_us(start_timestamp_us: i64, timestamp: gst::ClockTime) -> i64 { + let timestamp_us = timestamp.useconds().min(i64::MAX as u64) as i64; + start_timestamp_us.saturating_add(timestamp_us) +} + +/// Name of the appsink element the pipeline helpers look up or create. +pub const ENCODED_APPSINK_NAME: &str = "lk_appsink"; + +/// Error returned by the GStreamer pipeline helpers. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum GStreamerPipelineError { + /// The requested codec does not match what the pipeline advertises. + #[error( + "GStreamer codec mismatch: requested {requested:?}, but {location} advertises {advertised:?}" + )] + CodecMismatch { + /// Codec requested by the caller. + requested: EncodedVideoCodec, + /// Codec advertised by the pipeline. + advertised: EncodedVideoCodec, + /// Pipeline location that advertised the codec. + location: String, + }, + /// The pipeline has no usable appsink and no unlinked encoded pad. + #[error( + "GStreamer pipeline must include `appsink name={ENCODED_APPSINK_NAME}` or leave one \ + encoded video source pad unlinked" + )] + MissingAppSink, + /// The named element exists but is not an appsink. + #[error("GStreamer element {ENCODED_APPSINK_NAME} is not an appsink")] + NotAnAppSink, + /// Pad caps advertise no supported encoded video codec. + #[error("unlinked GStreamer pad '{0}' does not advertise supported encoded video caps")] + UnsupportedPadCaps(String), + /// Caps advertise a stream layout the encoded sources cannot consume. + #[error("unsupported GStreamer caps: {0}")] + UnsupportedCaps(String), + /// Element creation or linking failed. + #[error("{0}")] + Pipeline(String), +} + +/// Returns the appsink caps for a codec as a launch-string fragment. +/// +/// This is the single per-codec caps table: [`encoded_caps`] and pipeline +/// descriptions embedding a capsfilter should all derive from it. +pub fn encoded_caps_string(codec: EncodedVideoCodec) -> &'static str { + match codec { + EncodedVideoCodec::H264 => "video/x-h264,stream-format=byte-stream,alignment=au", + EncodedVideoCodec::H265 => "video/x-h265,stream-format=byte-stream,alignment=au", + EncodedVideoCodec::VP8 => "video/x-vp8", + EncodedVideoCodec::VP9 => "video/x-vp9,profile=(string)0", + EncodedVideoCodec::AV1 => "video/x-av1,stream-format=obu-stream,alignment=tu", + } +} + +/// Returns the appsink caps for a codec. +pub fn encoded_caps(codec: EncodedVideoCodec) -> Result { + encoded_caps_string(codec) + .parse::() + .map_err(|err| GStreamerPipelineError::Pipeline(format!("invalid encoded caps: {err}"))) +} + +/// Returns the appsink sample format used to ingest a codec. +pub fn sample_format_for_codec(codec: EncodedVideoCodec) -> GStreamerSampleFormat { + match codec { + EncodedVideoCodec::H264 => GStreamerSampleFormat::H264AnnexB, + EncodedVideoCodec::H265 => GStreamerSampleFormat::H265AnnexB, + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + GStreamerSampleFormat::AccessUnit { codec } + } + } +} + +/// Returns the parser element name used to normalize a codec, when one is needed. +pub fn parser_name(codec: EncodedVideoCodec) -> Option<&'static str> { + match codec { + EncodedVideoCodec::H264 => Some("h264parse"), + EncodedVideoCodec::H265 => Some("h265parse"), + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 => None, + EncodedVideoCodec::AV1 => Some("av1parse"), + } +} + +/// Finds or builds the encoded appsink in a pipeline. +/// +/// When the pipeline already contains `appsink name=lk_appsink`, it is used +/// as-is (its sink caps decide the sample format). Otherwise the pipeline +/// must leave one encoded video source pad unlinked; the codec parser, a +/// capsfilter, and an appsink are created and linked to it. +pub fn ensure_encoded_appsink( + pipeline: &gst::Pipeline, + requested_codec: Option, +) -> Result<(gst_app::AppSink, GStreamerSampleFormat), GStreamerPipelineError> { + if let Some(appsink) = pipeline.by_name(ENCODED_APPSINK_NAME) { + let sample_format = match sample_format_from_element_sink_caps(&appsink)? { + Some(sample_format) => { + if let Some(requested_codec) = requested_codec { + if requested_codec != sample_format.codec() { + return Err(GStreamerPipelineError::CodecMismatch { + requested: requested_codec, + advertised: sample_format.codec(), + location: format!("appsink '{ENCODED_APPSINK_NAME}'"), + }); + } + } + sample_format + } + None => sample_format_for_codec(requested_codec.unwrap_or(EncodedVideoCodec::H264)), + }; + let appsink = appsink + .downcast::() + .map_err(|_| GStreamerPipelineError::NotAnAppSink)?; + return Ok((appsink, sample_format)); + } + + let src_pad = pipeline + .find_unlinked_pad(gst::PadDirection::Src) + .ok_or(GStreamerPipelineError::MissingAppSink)?; + let inferred_codec = codec_from_pad_caps(&src_pad) + .ok_or_else(|| GStreamerPipelineError::UnsupportedPadCaps(src_pad.name().to_string()))?; + let codec = match requested_codec { + Some(requested_codec) if requested_codec != inferred_codec => { + return Err(GStreamerPipelineError::CodecMismatch { + requested: requested_codec, + advertised: inferred_codec, + location: format!("unlinked pad '{}'", src_pad.name()), + }); + } + Some(requested_codec) => requested_codec, + None => inferred_codec, + }; + let sample_format = sample_format_for_codec(codec); + let src_element = src_pad.parent_element().ok_or_else(|| { + GStreamerPipelineError::Pipeline( + "unlinked GStreamer encoded pad has no parent element".to_owned(), + ) + })?; + + let parser = parser_element_for_codec(codec)?; + let codec_caps = encoded_caps(codec)?; + let capsfilter = gst::ElementFactory::make("capsfilter") + .property("caps", codec_caps) + .build() + .map_err(|err| { + GStreamerPipelineError::Pipeline(format!("failed to create {codec:?} capsfilter: {err}")) + })?; + let appsink = gst::ElementFactory::make("appsink") + .name(ENCODED_APPSINK_NAME) + .property("sync", false) + .property("max-buffers", 8u32) + .property("drop", true) + .build() + .map_err(|err| { + GStreamerPipelineError::Pipeline(format!("failed to create appsink: {err}")) + })?; + + if let Some(parser) = &parser { + pipeline.add(parser).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to add {} to GStreamer pipeline: {err}", + parser.name() + )) + })?; + } + pipeline.add(&capsfilter).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to add capsfilter to GStreamer pipeline: {err}" + )) + })?; + pipeline.add(&appsink).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to add appsink to GStreamer pipeline: {err}" + )) + })?; + if let Some(parser) = &parser { + gst::Element::link_many([parser, &capsfilter, &appsink]).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to link {} to appsink: {err}", + parser.name() + )) + })?; + } else { + gst::Element::link_many([&capsfilter, &appsink]).map_err(|err| { + GStreamerPipelineError::Pipeline(format!("failed to link capsfilter to appsink: {err}")) + })?; + } + let link_target = parser.as_ref().unwrap_or(&capsfilter); + let sink_pad = link_target.static_pad("sink").ok_or_else(|| { + GStreamerPipelineError::Pipeline(format!( + "{} did not expose a sink pad", + link_target.name() + )) + })?; + src_pad.link(&sink_pad).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to link '{}' to {}: {err}", + src_element.name(), + link_target.name() + )) + })?; + + let appsink = + appsink.downcast::().map_err(|_| GStreamerPipelineError::NotAnAppSink)?; + Ok((appsink, sample_format)) +} + +fn parser_element_for_codec( + codec: EncodedVideoCodec, +) -> Result, GStreamerPipelineError> { + let Some(name) = parser_name(codec) else { + return Ok(None); + }; + let mut builder = gst::ElementFactory::make(name); + if matches!(codec, EncodedVideoCodec::H264 | EncodedVideoCodec::H265) { + builder = builder.property("config-interval", -1i32); + } + builder + .build() + .map(Some) + .map_err(|err| GStreamerPipelineError::Pipeline(format!("failed to create {name}: {err}"))) +} + +fn sample_format_from_element_sink_caps( + element: &gst::Element, +) -> Result, GStreamerPipelineError> { + let Some(sink_pad) = element.static_pad("sink") else { + return Ok(None); + }; + sample_format_from_pad_caps(&sink_pad) +} + +fn sample_format_from_pad_caps( + pad: &gst::Pad, +) -> Result, GStreamerPipelineError> { + let caps = pad.current_caps().unwrap_or_else(|| pad.query_caps(None)); + for structure in caps.iter() { + if let Some(sample_format) = sample_format_from_caps_structure(structure)? { + return Ok(Some(sample_format)); + } + } + Ok(None) +} + +/// Infers the appsink sample format from a caps structure. +pub fn sample_format_from_caps_structure( + structure: &gst::StructureRef, +) -> Result, GStreamerPipelineError> { + let Some(codec) = codec_from_caps_name(structure.name()) else { + return Ok(None); + }; + + match codec { + EncodedVideoCodec::H264 => { + let stream_format = structure.get::("stream-format").ok(); + match stream_format.as_deref() { + Some("avc") | Some("avc3") => Ok(Some(GStreamerSampleFormat::H264Avc { + nal_length_size: h264_avc_nal_length_size_from_caps(structure), + })), + Some("byte-stream") | None => Ok(Some(GStreamerSampleFormat::H264AnnexB)), + Some(stream_format) => Err(GStreamerPipelineError::UnsupportedCaps(format!( + "H.264 stream-format '{stream_format}'; expected byte-stream or avc" + ))), + } + } + EncodedVideoCodec::H265 => Ok(Some(GStreamerSampleFormat::H265AnnexB)), + EncodedVideoCodec::VP8 => Ok(Some(GStreamerSampleFormat::AccessUnit { codec })), + EncodedVideoCodec::VP9 => { + let profile = structure.get::("profile").ok(); + match profile.as_deref() { + Some("0") | None => Ok(Some(GStreamerSampleFormat::AccessUnit { codec })), + Some(profile) => Err(GStreamerPipelineError::UnsupportedCaps(format!( + "VP9 profile '{profile}'; expected profile 0" + ))), + } + } + EncodedVideoCodec::AV1 => { + let stream_format = structure.get::("stream-format").ok(); + match stream_format.as_deref() { + Some("obu-stream") | None => Ok(Some(GStreamerSampleFormat::AccessUnit { codec })), + Some(stream_format) => Err(GStreamerPipelineError::UnsupportedCaps(format!( + "AV1 stream-format '{stream_format}'; expected obu-stream" + ))), + } + } + } +} + +fn h264_avc_nal_length_size_from_caps(structure: &gst::StructureRef) -> u8 { + let Ok(codec_data) = structure.get::("codec_data") else { + return 4; + }; + let Ok(codec_data) = codec_data.map_readable() else { + return 4; + }; + h264_avc_nal_length_size_from_codec_data(codec_data.as_ref()).unwrap_or(4) +} + +/// Reads the AVC NAL length-prefix size from `avcC` codec data. +pub fn h264_avc_nal_length_size_from_codec_data(codec_data: &[u8]) -> Option { + let length_size = (codec_data.get(4)? & 0x03) + 1; + (1..=4).contains(&length_size).then_some(length_size) +} + +/// Infers the encoded codec advertised by a pad's caps. +pub fn codec_from_pad_caps(pad: &gst::Pad) -> Option { + let caps = pad.current_caps().unwrap_or_else(|| pad.query_caps(None)); + caps.iter().find_map(|structure| codec_from_caps_name(structure.name())) +} + +/// Maps a caps media-type name to an encoded codec. +pub fn codec_from_caps_name(name: &str) -> Option { + match name { + "video/x-h264" => Some(EncodedVideoCodec::H264), + "video/x-h265" => Some(EncodedVideoCodec::H265), + "video/x-vp8" => Some(EncodedVideoCodec::VP8), + "video/x-vp9" => Some(EncodedVideoCodec::VP9), + "video/x-av1" => Some(EncodedVideoCodec::AV1), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sample_payload_h264_annex_b_detects_keyframe() { + let access_unit = access_unit_from_sample_payload( + GStreamerSampleFormat::H264AnnexB, + &[0, 0, 1, 0x65, 1, 2], + 1_000, + EncodedFrameType::Delta, + 640, + 480, + ) + .unwrap(); + + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.timestamp_us, 1_000); + } + + #[test] + fn sample_payload_h264_avc_converts_to_annex_b_and_detects_keyframe() { + let access_unit = access_unit_from_sample_payload( + GStreamerSampleFormat::H264Avc { nal_length_size: 4 }, + &[0, 0, 0, 3, 0x65, 1, 2], + 1_000, + EncodedFrameType::Delta, + 640, + 480, + ) + .unwrap(); + + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + } + + #[test] + fn sample_payload_access_unit_uses_buffer_delta_flag() { + let access_unit = access_unit_from_sample_payload( + GStreamerSampleFormat::AccessUnit { codec: EncodedVideoCodec::VP8 }, + &[1, 2, 3], + 2_000, + EncodedFrameType::Delta, + 640, + 480, + ) + .unwrap(); + + assert_eq!(access_unit.codec, EncodedVideoCodec::VP8); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!( + access_unit.codec_specific, + CodecSpecific::VP8 { temporal_id: None, layer_sync: false } + ); + } + + #[test] + fn sample_payload_access_unit_sets_vp9_and_av1_specifics() { + let vp9 = access_unit_from_sample_payload( + GStreamerSampleFormat::AccessUnit { codec: EncodedVideoCodec::VP9 }, + &[1, 2, 3], + 2_000, + EncodedFrameType::Key, + 640, + 480, + ) + .unwrap(); + assert_eq!(vp9.codec_specific, CodecSpecific::default_for(EncodedVideoCodec::VP9)); + + let av1 = access_unit_from_sample_payload( + GStreamerSampleFormat::AccessUnit { codec: EncodedVideoCodec::AV1 }, + &[1, 2, 3], + 2_000, + EncodedFrameType::Key, + 640, + 480, + ) + .unwrap(); + assert_eq!(av1.codec_specific, CodecSpecific::default_for(EncodedVideoCodec::AV1)); + } + + #[test] + fn clock_time_is_offset_from_start_timestamp() { + let timestamp = clock_time_to_timestamp_us(10_000, gst::ClockTime::from_useconds(1_234)); + assert_eq!(timestamp, 11_234); + } +} diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs new file mode 100644 index 000000000..307f844d1 --- /dev/null +++ b/livekit-capture/src/sources/mod.rs @@ -0,0 +1,18 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Optional capture sources that feed the shared capture paths. + +#[cfg(feature = "gstreamer")] +pub mod gstreamer; From 0ac5cd86cadd3ec15391edb2c75de913d5fe2df3 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:48:45 -0700 Subject: [PATCH 08/56] Add readme and changeset for livekit-capture --- .changeset/livekit-capture.md | 5 +++++ livekit-capture/README.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 .changeset/livekit-capture.md create mode 100644 livekit-capture/README.md diff --git a/.changeset/livekit-capture.md b/.changeset/livekit-capture.md new file mode 100644 index 000000000..2ef99a76d --- /dev/null +++ b/.changeset/livekit-capture.md @@ -0,0 +1,5 @@ +--- +"livekit-capture": minor +--- + +Add a `livekit-capture` crate with codec-neutral encoded capture types, H264/H265/VP8/VP9/AV1 passthrough support, common encoded ingress helpers, and GStreamer appsink encoded ingress. Encoded sources honor WebRTC rate-control targets, validate pre-encoded AV1 and H265 access units on ingest, and support opt-in frame metadata for capture latency measurement. diff --git a/livekit-capture/README.md b/livekit-capture/README.md new file mode 100644 index 000000000..eadf7a029 --- /dev/null +++ b/livekit-capture/README.md @@ -0,0 +1,31 @@ +# livekit-capture + +Helpers for publishing pre-encoded video with the LiveKit Rust SDK. The +optional `gstreamer` feature turns a GStreamer `appsink` into an encoded +ingest source. + +## Library entry points + +- `VideoCaptureTrack::new` for decoded-frame publishing and + `VideoCaptureTrack::new_encoded` for pre-encoded passthrough (no raw + keepalive frames, so the sender starts directly on the passthrough encoder). +- `EncodedIngress` — the pre-encoded pump used when the caller manages its + own source: `capture_next()` reports each published access unit, + `stop_handle()` cancels from any thread, and downstream keyframe requests + (PLI/FIR) are forwarded to the source automatically. Passthrough is + single-layer (`L1T1`), and access units carrying other layering metadata are + rejected. +- `sources::gstreamer::ensure_encoded_appsink` and friends turn an arbitrary + pipeline (containing `appsink name=lk_appsink` or one unlinked encoded pad) + into an encoded source; `encoded_caps_string` is the single per-codec caps + table. The GStreamer source answers keyframe requests with a + `GstForceKeyUnit` upstream event. + +## GStreamer ingest + +`GStreamerAppSinkEncodedSource` implements `EncodedAccessUnitSource` on top of +an `appsink` producing H.264 (Annex-B or AVC), H.265 Annex-B, VP8, VP9, or +AV1 access units. Feed it to `EncodedIngress` together with a +`VideoCaptureTrack::new_encoded` track, then publish the track with +`VideoCaptureTrack::encoded_publish_options(codec)` so the sender uses the +pre-encoded passthrough encoder. From 95f4fc9644810e404ebaeeb11ca659f295389ba0 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:49:24 -0700 Subject: [PATCH 09/56] Update cargo lock --- Cargo.lock | 238 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 223 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edecdcd92..9a02942c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -549,6 +549,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomic_refcell" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" + [[package]] name = "autocfg" version = "1.5.0" @@ -568,7 +574,7 @@ dependencies = [ "log", "num-rational", "num-traits", - "pastey", + "pastey 0.1.1", "rayon", "thiserror 2.0.18", "v_frame", @@ -2694,8 +2700,21 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0071fe88dba8e40086c8ff9bbb62622999f49628344b1d1bf490a48a29d80f22" dependencies = [ - "glib-sys", - "gobject-sys", + "glib-sys 0.21.5", + "gobject-sys 0.21.5", + "libc", + "system-deps", + "windows-sys 0.61.2", +] + +[[package]] +name = "gio-sys" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353fdc7da7cd16da916104b1e0e4e7de380ec9c8aaa20d4d742d66310ab4b0d5" +dependencies = [ + "glib-sys 0.22.8", + "gobject-sys 0.22.6", "libc", "system-deps", "windows-sys 0.61.2", @@ -2724,10 +2743,31 @@ dependencies = [ "futures-executor", "futures-task", "futures-util", - "gio-sys", - "glib-macros", - "glib-sys", - "gobject-sys", + "gio-sys 0.21.5", + "glib-macros 0.21.5", + "glib-sys 0.21.5", + "gobject-sys 0.21.5", + "libc", + "memchr", + "smallvec", +] + +[[package]] +name = "glib" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddbcf514bd1881fc1b960e4e52b4e82873f4da3bceddbd58d42827b508888100" +dependencies = [ + "bitflags 2.13.0", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "futures-util", + "gio-sys 0.22.8", + "glib-macros 0.22.6", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", "libc", "memchr", "smallvec", @@ -2746,6 +2786,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "glib-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "506d23499707c7142898429757e8d9a3871d965239a2cb66dfa05052be6d6f19" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "glib-sys" version = "0.21.5" @@ -2756,6 +2808,16 @@ dependencies = [ "system-deps", ] +[[package]] +name = "glib-sys" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030967459f9f676851872c6304adea7825c6d462ec9b72554c733cf0c5952233" +dependencies = [ + "libc", + "system-deps", +] + [[package]] name = "glifo" version = "0.1.1" @@ -2817,7 +2879,18 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dca35da0d19a18f4575f3cb99fe1c9e029a2941af5662f326f738a21edaf294" dependencies = [ - "glib-sys", + "glib-sys 0.21.5", + "libc", + "system-deps", +] + +[[package]] +name = "gobject-sys" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a861859b887a79cf461359c192c97a57d8fb0229dd291232e57aa11f6fa72c" +dependencies = [ + "glib-sys 0.22.8", "libc", "system-deps", ] @@ -2867,6 +2940,99 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "gstreamer" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4527e1b9bae8d29ce137bde5b8eec8ae8f78f13ad00fc6e70cbe227d6ad027" +dependencies = [ + "cfg-if 1.0.4", + "futures-channel", + "futures-core", + "futures-util", + "glib 0.22.8", + "gstreamer-sys", + "itertools 0.15.0", + "kstring", + "libc", + "muldiv", + "num-integer", + "num-rational", + "option-operations", + "pastey 0.2.3", + "pin-project-lite", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gstreamer-app" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97f8ae9238c2352398dcc084de28df3f7099af216ac6c160b52318d23f25c010" +dependencies = [ + "futures-core", + "futures-sink", + "glib 0.22.8", + "gstreamer", + "gstreamer-app-sys", + "gstreamer-base", + "libc", +] + +[[package]] +name = "gstreamer-app-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a74a8211e5d7df2f45b612c284ddf56b92bdf4e879e8ed72e7c46dd0842e158" +dependencies = [ + "glib-sys 0.22.8", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-base" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91c94a4d3047d05dd6e1f6d91c74f61f56384c7ea1c9d0c1051572eeeb0138d" +dependencies = [ + "atomic_refcell", + "cfg-if 1.0.4", + "glib 0.22.8", + "gstreamer", + "gstreamer-base-sys", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fbbc623dc066908ba10c43d629c21096508dea04796592a206c4edd864e37" +dependencies = [ + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533fa8d28fc830eafccbcfcfddb390563ea5d3a351af2c3aab99e197e5f5b1ba" +dependencies = [ + "cfg-if 1.0.4", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "libc", + "system-deps", +] + [[package]] name = "guillotiere" version = "0.7.0" @@ -3205,7 +3371,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -3528,6 +3694,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3792,7 +3967,7 @@ version = "0.3.41" dependencies = [ "cxx", "env_logger 0.11.10", - "glib", + "glib 0.21.5", "jni 0.21.1", "js-sys", "lazy_static", @@ -3987,6 +4162,18 @@ dependencies = [ "url", ] +[[package]] +name = "livekit-capture" +version = "0.1.0" +dependencies = [ + "bytes", + "gstreamer", + "gstreamer-app", + "livekit", + "log", + "thiserror 2.0.18", +] + [[package]] name = "livekit-common" version = "0.1.0" @@ -4359,6 +4546,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "muldiv" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0" + [[package]] name = "multimap" version = "0.10.1" @@ -5280,6 +5473,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-operations" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aca39cf52b03268400c16eeb9b56382ea3c3353409309b63f5c8f0b1faf42754" +dependencies = [ + "pastey 0.2.3", +] + [[package]] name = "orbclient" version = "0.3.51" @@ -5396,6 +5598,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pbjson" version = "0.6.0" @@ -5926,7 +6134,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -5963,7 +6171,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -7159,7 +7367,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand 2.3.0", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -7729,7 +7937,7 @@ dependencies = [ "num-complex", "num-integer", "num-traits", - "pastey", + "pastey 0.1.1", "rustfft", "smallvec", "tract-data", @@ -7792,7 +8000,7 @@ dependencies = [ "liquid-derive", "log", "num-traits", - "pastey", + "pastey 0.1.1", "scan_fmt", "smallvec", "time", From 3c04826b8ff514f9571e872570a65439ec94408f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:15:34 -0700 Subject: [PATCH 10/56] Replace `VideoCaptureTrack` with extension trait `VideoCaptureTrack` has tight coupling with RTC-level APIs, owns track for no reason, and confuses the concept of "track" and "source." The extension trait [pattern](http://xion.io/post/code/rust-extension-traits.html) is a better fit here. --- livekit-capture/src/encoded/ingress.rs | 51 ++++++++------ livekit-capture/src/lib.rs | 1 - livekit-capture/src/track.rs | 97 +++++++------------------- 3 files changed, 53 insertions(+), 96 deletions(-) diff --git a/livekit-capture/src/encoded/ingress.rs b/livekit-capture/src/encoded/ingress.rs index de1864e5a..1513bd5ce 100644 --- a/livekit-capture/src/encoded/ingress.rs +++ b/livekit-capture/src/encoded/ingress.rs @@ -21,12 +21,12 @@ use std::{ }, }; -use livekit::webrtc::video_frame::FrameMetadata; +use livekit::webrtc::{video_frame::FrameMetadata, video_source::native::NativeVideoSource}; use crate::{ encoded::{EncodedFrameType, EncodedRateControl, OwnedEncodedAccessUnit}, error::CaptureError, - track::VideoCaptureTrack, + track::NativeVideoSourceExt, }; /// Source of owned encoded access units. @@ -109,16 +109,21 @@ impl EncodedIngressStop { /// Pulls encoded access units from a source and forwards them into a video track. #[derive(Debug)] pub struct EncodedIngress { - track: VideoCaptureTrack, - source: S, + rtc_source: NativeVideoSource, + capture_source: S, stop: EncodedIngressStop, awaiting_initial_keyframe: bool, } impl EncodedIngress { /// Creates an encoded ingress runner. - pub fn new(track: VideoCaptureTrack, source: S) -> Self { - Self { track, source, stop: EncodedIngressStop::new(), awaiting_initial_keyframe: true } + pub fn new(rtc_source: NativeVideoSource, capture_source: S) -> Self { + Self { + rtc_source, + capture_source, + stop: EncodedIngressStop::new(), + awaiting_initial_keyframe: true, + } } /// Returns a cancellation handle for this runner. @@ -126,24 +131,24 @@ impl EncodedIngress { self.stop.clone() } - /// Returns the capture track used by this runner. - pub fn track(&self) -> &VideoCaptureTrack { - &self.track + /// Returns the RTC source used by this runner. + pub fn rtc_source(&self) -> &NativeVideoSource { + &self.rtc_source } /// Returns the underlying encoded source. pub fn source(&self) -> &S { - &self.source + &self.capture_source } /// Returns the underlying encoded source mutably. pub fn source_mut(&mut self) -> &mut S { - &mut self.source + &mut self.capture_source } /// Consumes this runner and returns its parts. - pub fn into_parts(self) -> (VideoCaptureTrack, S) { - (self.track, self.source) + pub fn into_parts(self) -> (NativeVideoSource, S) { + (self.rtc_source, self.capture_source) } } @@ -182,16 +187,16 @@ where &mut self, frame_metadata: impl FnOnce(&OwnedEncodedAccessUnit) -> Option, ) -> Result, EncodedIngressError> { - if let Some(rate_control) = self.track.take_rate_control_request() { - self.source.update_rate_control(rate_control); + if let Some(rate_control) = self.rtc_source.take_rate_control_request() { + self.capture_source.update_rate_control(rate_control); } - if self.track.take_keyframe_request() { - self.source.request_keyframe(); + if self.rtc_source.take_keyframe_request() { + self.capture_source.request_keyframe(); } let access_unit = loop { let Some(access_unit) = - self.source.next_access_unit().map_err(EncodedIngressError::Source)? + self.capture_source.next_access_unit().map_err(EncodedIngressError::Source)? else { return Ok(None); }; @@ -203,7 +208,7 @@ where }; let frame_metadata = frame_metadata(&access_unit); - self.track + self.rtc_source .capture_encoded_with_metadata(&access_unit.as_access_unit(), frame_metadata) .map_err(EncodedIngressError::Capture)?; Ok(Some(EncodedIngressCapture { @@ -274,8 +279,8 @@ mod tests { ) } - fn encoded_track() -> VideoCaptureTrack { - VideoCaptureTrack::new_encoded("test", VideoResolution { width: 640, height: 480 }) + fn rtc_source() -> NativeVideoSource { + NativeVideoSource::new_encoded(VideoResolution { width: 640, height: 480 }) } #[test] @@ -285,7 +290,7 @@ mod tests { access_unit(2, EncodedFrameType::Delta), access_unit(3, EncodedFrameType::Key), ]); - let mut ingress = EncodedIngress::new(encoded_track(), source); + let mut ingress = EncodedIngress::new(rtc_source(), source); let capture = ingress .capture_next() @@ -302,7 +307,7 @@ mod tests { access_unit(1, EncodedFrameType::Key), access_unit(2, EncodedFrameType::Delta), ]); - let mut ingress = EncodedIngress::new(encoded_track(), source); + let mut ingress = EncodedIngress::new(rtc_source(), source); let first = ingress.capture_next().unwrap().unwrap(); let second = ingress.capture_next().unwrap().unwrap(); diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index bbc3b339d..1fd1cd602 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -29,4 +29,3 @@ pub use encoded::{ H264PacketizationMode, OwnedEncodedAccessUnit, }; pub use error::CaptureError; -pub use track::VideoCaptureTrack; diff --git a/livekit-capture/src/track.rs b/livekit-capture/src/track.rs index fb825e335..2bc5bbfcb 100644 --- a/livekit-capture/src/track.rs +++ b/livekit-capture/src/track.rs @@ -29,60 +29,34 @@ use crate::{ error::CaptureError, }; -/// Capture source backed by a LiveKit local video track. -#[derive(Debug, Clone)] -pub struct VideoCaptureTrack { - source: NativeVideoSource, - track: LocalVideoTrack, -} - -impl VideoCaptureTrack { - /// Creates a capture track with the supplied resolution. - pub fn new(name: &str, resolution: VideoResolution, is_screencast: bool) -> Self { - let source = NativeVideoSource::new(resolution, is_screencast); - let track = - LocalVideoTrack::create_video_track(name, RtcVideoSource::Native(source.clone())); - Self { source, track } - } - - /// Creates a capture track for pre-encoded access units. - /// - /// Unlike [`VideoCaptureTrack::new`], no raw keepalive frames are - /// injected before the first capture, so the sender starts directly on - /// the passthrough encoder instead of briefly encoding black frames. - pub fn new_encoded(name: &str, resolution: VideoResolution) -> Self { - let source = NativeVideoSource::new_encoded(resolution); - let track = - LocalVideoTrack::create_video_track(name, RtcVideoSource::Native(source.clone())); - Self { source, track } - } - - /// Returns the publishable local video track. - pub fn track(&self) -> LocalVideoTrack { - self.track.clone() - } - - /// Captures one decoded video frame. - pub fn capture_frame>(&self, frame: &VideoFrame) { - self.source.capture_frame(frame); - } - +/// Additional methods for [`NativeVideoSource`] to support capture from sources. +pub trait NativeVideoSourceExt { /// Captures one encoded video access unit. /// /// The passthrough path forwards single-layer streams: access units /// carrying temporal/spatial layer ids, an AV1 dependency descriptor, or /// a non-`L1T1` scalability mode are rejected so callers are not misled /// into thinking that metadata reaches the wire. - pub fn capture_encoded(&self, access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError> { - self.capture_encoded_with_metadata(access_unit, None) - } + fn capture_encoded(&self, access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError>; /// Captures one encoded video access unit with optional frame metadata. /// /// Metadata is only propagated to subscribers when the corresponding /// [`TrackPublishOptions::frame_metadata_features`] are enabled before /// publishing the local track. - pub fn capture_encoded_with_metadata( + fn capture_encoded_with_metadata( + &self, + access_unit: &EncodedAccessUnit<'_>, + frame_metadata: Option, + ) -> Result<(), CaptureError>; +} + +impl NativeVideoSourceExt for NativeVideoSource { + fn capture_encoded(&self, access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError> { + self.capture_encoded_with_metadata(access_unit, None) + } + + fn capture_encoded_with_metadata( &self, access_unit: &EncodedAccessUnit<'_>, frame_metadata: Option, @@ -106,38 +80,17 @@ impl VideoCaptureTrack { resolution: VideoResolution { width: access_unit.width, height: access_unit.height }, frame_metadata, }; - self.source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) - } - - /// Returns and clears the pending keyframe request raised by the - /// passthrough encoder (PLI/FIR from the SFU, late subscriber join, or - /// sender reconfiguration). - /// - /// Poll this from the capture loop and forward the request to the - /// upstream encoder so it produces an IDR; until one arrives, new - /// subscribers cannot render the track. - pub fn take_keyframe_request(&self) -> bool { - self.source.take_keyframe_request() - } - - /// Returns and clears the pending rate-control target raised by the - /// passthrough encoder. - /// - /// Poll this from the capture loop and forward the target to the - /// upstream encoder so congestion control can adjust the produced - /// bitrate. - pub fn take_rate_control_request(&self) -> Option { - self.source.take_rate_control_request() + self.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) } +} - /// Returns publish options appropriate for encoded passthrough. - pub fn encoded_publish_options(codec: EncodedVideoCodec) -> TrackPublishOptions { - TrackPublishOptions { - video_codec: codec.into(), - video_encoder: VideoEncoderBackend::PreEncoded, - simulcast: false, - ..Default::default() - } +/// Returns publish options appropriate for encoded passthrough. +pub fn encoded_publish_options(codec: EncodedVideoCodec) -> TrackPublishOptions { + TrackPublishOptions { + video_codec: codec.into(), + video_encoder: VideoEncoderBackend::PreEncoded, + simulcast: false, + ..Default::default() } } From 23a4c90f7bbe133c673715040c52f8f18500d392 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:28:22 -0700 Subject: [PATCH 11/56] Remove unused imports --- livekit-capture/src/track.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/livekit-capture/src/track.rs b/livekit-capture/src/track.rs index 2bc5bbfcb..ab41f5352 100644 --- a/livekit-capture/src/track.rs +++ b/livekit-capture/src/track.rs @@ -14,17 +14,15 @@ use livekit::{ options::{TrackPublishOptions, VideoEncoderBackend}, - prelude::LocalVideoTrack, webrtc::{ - video_frame::{EncodedVideoFrame, FrameMetadata, VideoBuffer, VideoFrame}, - video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution}, + video_frame::{EncodedVideoFrame, FrameMetadata}, + video_source::{native::NativeVideoSource, VideoResolution}, }, }; use crate::{ encoded::{ - CodecSpecific, EncodedAccessUnit, EncodedLayerInfo, EncodedPayload, EncodedRateControl, - EncodedVideoCodec, + CodecSpecific, EncodedAccessUnit, EncodedLayerInfo, EncodedPayload, EncodedVideoCodec, }, error::CaptureError, }; From dff569c7fdde41881a38a85822fb03f02bccf84b Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:12:58 -0700 Subject: [PATCH 12/56] Update readme for extension trait --- livekit-capture/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index eadf7a029..d912f5176 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -6,8 +6,9 @@ ingest source. ## Library entry points -- `VideoCaptureTrack::new` for decoded-frame publishing and - `VideoCaptureTrack::new_encoded` for pre-encoded passthrough (no raw +- `track::NativeVideoSourceExt` — extension methods on the RTC-level + `NativeVideoSource` for capturing pre-encoded access units. Use + `NativeVideoSource::new_encoded` for pre-encoded passthrough (no raw keepalive frames, so the sender starts directly on the passthrough encoder). - `EncodedIngress` — the pre-encoded pump used when the caller manages its own source: `capture_next()` reports each published access unit, @@ -26,6 +27,6 @@ ingest source. `GStreamerAppSinkEncodedSource` implements `EncodedAccessUnitSource` on top of an `appsink` producing H.264 (Annex-B or AVC), H.265 Annex-B, VP8, VP9, or AV1 access units. Feed it to `EncodedIngress` together with a -`VideoCaptureTrack::new_encoded` track, then publish the track with -`VideoCaptureTrack::encoded_publish_options(codec)` so the sender uses the -pre-encoded passthrough encoder. +`NativeVideoSource::new_encoded` RTC source, then publish a local video track +created from that source with `track::encoded_publish_options(codec)` so the +sender uses the pre-encoded passthrough encoder. From f9bb450da864bba7270e17f19174706b6b55ce67 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:26:43 -0700 Subject: [PATCH 13/56] Fix dead code warnings in h26x module --- livekit-capture/src/encoded/h26x.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index 7ee5335d3..d33567655 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -32,6 +32,7 @@ const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; /// `push` appends bytes and returns at most one completed access unit; call /// `drain` repeatedly to pull further access units already buffered, and /// `flush` once at end of stream to emit the final pending access unit. +#[cfg(test)] pub(crate) trait AccessUnitParser { /// Appends bytes and returns the next complete access unit, if any. fn push(&mut self, bytes: &[u8]) -> Result, CaptureError>; @@ -62,7 +63,7 @@ pub struct AnnexBAccessUnitParser { } /// H.264/AVC length-prefixed parser state. -#[cfg(any(feature = "tcpsink", test))] +#[cfg(test)] #[derive(Debug, Clone)] pub(crate) struct AvcAccessUnitParser { pending: Vec, @@ -194,6 +195,7 @@ impl AnnexBAccessUnitParser { } } +#[cfg(test)] impl AccessUnitParser for AnnexBAccessUnitParser { fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { AnnexBAccessUnitParser::push(self, bytes) @@ -204,7 +206,7 @@ impl AccessUnitParser for AnnexBAccessUnitParser { } } -#[cfg(any(feature = "tcpsink", test))] +#[cfg(test)] impl AvcAccessUnitParser { /// Creates a parser for H.264/AVC length-prefixed byte streams. pub(crate) fn new( @@ -328,7 +330,7 @@ impl AvcAccessUnitParser { } } -#[cfg(any(feature = "tcpsink", test))] +#[cfg(test)] impl AccessUnitParser for AvcAccessUnitParser { fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { AvcAccessUnitParser::push(self, bytes) @@ -376,7 +378,7 @@ pub fn annex_b_nalus(bytes: &[u8]) -> Result, CaptureError> { } /// Creates an Annex-B access unit from H.264/AVC length-prefixed NAL units. -pub(crate) fn access_unit_from_h264_avc( +pub fn access_unit_from_h264_avc( payload: &[u8], nal_length_size: u8, timestamp_us: i64, @@ -439,7 +441,7 @@ fn access_unit_split_index( } } -#[cfg(any(feature = "tcpsink", test))] +#[cfg(test)] fn avc_access_unit_split_index( bytes: &[u8], ranges: &[Range], From 1f380f73829a95874f4d261f33d569c2f999cbc5 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:52:36 -0700 Subject: [PATCH 14/56] Use conventional mod structure --- livekit-capture/src/{encoded.rs => encoded/mod.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename livekit-capture/src/{encoded.rs => encoded/mod.rs} (100%) diff --git a/livekit-capture/src/encoded.rs b/livekit-capture/src/encoded/mod.rs similarity index 100% rename from livekit-capture/src/encoded.rs rename to livekit-capture/src/encoded/mod.rs From 1eae8087c9ca0003db657ac1d1fd6b33601b0aa5 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:51:03 -0700 Subject: [PATCH 15/56] Wip --- Cargo.lock | 1 + livekit-capture/Cargo.toml | 5 + livekit-capture/src/error.rs | 3 + livekit-capture/src/lib.rs | 14 +- livekit-capture/src/pump.rs | 648 ++++++++++++++++++++++++++++ livekit-capture/src/source.rs | 182 ++++++++ livekit-capture/src/sources/demo.rs | 232 ++++++++++ livekit-capture/src/sources/mod.rs | 3 + 8 files changed, 1087 insertions(+), 1 deletion(-) create mode 100644 livekit-capture/src/pump.rs create mode 100644 livekit-capture/src/source.rs create mode 100644 livekit-capture/src/sources/demo.rs diff --git a/Cargo.lock b/Cargo.lock index 9a02942c0..fbd0a6461 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4172,6 +4172,7 @@ dependencies = [ "livekit", "log", "thiserror 2.0.18", + "tokio", ] [[package]] diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index d82996f09..d6b3f6e69 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -14,7 +14,12 @@ gstreamer-app = { version = "0.25.2", optional = true } livekit = { workspace = true } log = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true, features = ["sync"] } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] default = [] +demo = [] gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index 6b12d8a46..47be05d3f 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -34,6 +34,9 @@ pub enum CaptureError { /// Encoded payload or transport data is malformed. #[error("invalid encoded data: {0}")] InvalidEncodedData(&'static str), + /// Pixel frame data is malformed. + #[error("invalid pixel frame: {0}")] + InvalidPixelFrame(&'static str), /// Wire format is represented by the API but not supported by this source. #[error("encoded wire format is not supported by this source: {0:?}")] UnsupportedWireFormat(EncodedWireFormat), diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 1fd1cd602..fcfef51f4 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -12,13 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Helpers for publishing pre-encoded video with LiveKit. +//! Capture sources and helpers for publishing video with LiveKit. pub mod encoded; mod error; +pub mod pump; +pub mod source; pub mod sources; pub mod track; +pub use pump::{ + RunningVideoPump, VideoPump, VideoPumpError, VideoPumpExit, VideoPumpStats, VideoPumpStop, +}; +pub use source::{ + EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, + SourceError, VideoResolution, VideoSource, +}; +#[cfg(feature = "demo")] +pub use sources::demo::{DemoSource, DemoSourceConfig}; + pub use encoded::{ ingress::{ EncodedAccessUnitSource, EncodedIngress, EncodedIngressCapture, EncodedIngressError, diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs new file mode 100644 index 000000000..a61f01810 --- /dev/null +++ b/livekit-capture/src/pump.rs @@ -0,0 +1,648 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pumps frames from a capture source into an RTC video source. +//! +//! [`VideoPump`] is the bridge between the libwebrtc-free source traits in +//! [`source`](crate::source) and a publishable RTC track: it builds the +//! matching [`NativeVideoSource`], converts crate-owned frame types at the +//! boundary, and forwards downstream keyframe and rate-control requests back +//! to encoded sources. + +use std::{ + any::Any, + io, + panic::{catch_unwind, AssertUnwindSafe}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, +}; + +use livekit::{ + options::{TrackPublishOptions, VideoEncoderBackend}, + webrtc::{ + video_frame::{EncodedVideoFrame, I420Buffer, VideoFrame, VideoRotation}, + video_source::{ + native::NativeVideoSource, EncodedRateControl, RtcVideoSource, + VideoResolution as RtcVideoResolution, + }, + }, +}; +use thiserror::Error; + +use crate::{ + encoded::{CodecSpecific, EncodedFrameType, EncodedLayerInfo, OwnedEncodedAccessUnit}, + error::CaptureError, + source::{ + EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, + SourceError, VideoResolution, VideoSource, + }, +}; + +impl From for RateControl { + fn from(target: EncodedRateControl) -> Self { + Self { target_bitrate_bps: target.target_bitrate_bps, framerate_fps: target.framerate_fps } + } +} + +impl From for RtcVideoResolution { + fn from(resolution: VideoResolution) -> Self { + Self { width: resolution.width, height: resolution.height } + } +} + +/// Error returned by a pump run. +#[derive(Debug, Error)] +pub enum VideoPumpError { + /// The capture source failed. + #[error("capture source failed")] + Source(#[from] SourceError), + /// The RTC source rejected a frame. + #[error("frame capture failed")] + Capture(#[from] CaptureError), + /// The pump thread panicked. + #[error("pump panicked: {0}")] + Panicked(String), +} + +/// Why a pump run ended successfully. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoPumpExit { + /// The stop handle was fired. + Stopped, + /// The source reached the end of its stream. + EndOfStream, +} + +/// Statistics returned when a pump run ends. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct VideoPumpStats { + /// Number of frames or access units captured. + pub frames_captured: u64, + /// Why the run ended. + pub exit: VideoPumpExit, +} + +/// Cancellation handle for a [`VideoPump`]. +/// +/// Cheap to clone; wire it to a shutdown signal and call +/// [`VideoPumpStop::stop`] from any thread to make the pump return after the +/// frame in flight. +#[derive(Debug, Clone, Default)] +pub struct VideoPumpStop(Arc); + +impl VideoPumpStop { + /// Creates an un-stopped handle. + pub fn new() -> Self { + Self::default() + } + + /// Signals the pump to stop. + pub fn stop(&self) { + self.0.store(true, Ordering::Release); + } + + /// Returns true once [`VideoPumpStop::stop`] has been called. + pub fn is_stopped(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + +/// Pumps a [`VideoSource`] into an RTC video source. +/// +/// The pump owns every libwebrtc interaction: it builds the RTC source +/// appropriate for the capture source kind, derives the matching publish +/// options, converts frames at the boundary, and polls downstream keyframe +/// and rate-control requests, forwarding them to encoded sources as +/// crate-owned types. +#[derive(Debug)] +pub struct VideoPump { + source: VideoSource, + rtc_source: NativeVideoSource, + stop: VideoPumpStop, +} + +impl VideoPump { + /// Creates a pump for a capture source, building the matching RTC source. + /// + /// For pixel sources this must be called from the context of the async + /// runtime driving the SDK, because the RTC source spawns its keepalive + /// task at construction. The pump itself runs on plain threads. + pub fn new(source: impl Into) -> Self { + let source = source.into(); + let resolution = source.resolution().into(); + let rtc_source = match &source { + VideoSource::Pixel(_) => NativeVideoSource::new(resolution, false), + VideoSource::Encoded(_) => NativeVideoSource::new_encoded(resolution), + }; + Self { source, rtc_source, stop: VideoPumpStop::new() } + } + + /// Returns the RTC source to create the local track with. + pub fn rtc_source(&self) -> RtcVideoSource { + RtcVideoSource::Native(self.rtc_source.clone()) + } + + /// Returns publish options appropriate for the source kind. + pub fn publish_options(&self) -> TrackPublishOptions { + match &self.source { + VideoSource::Pixel(_) => TrackPublishOptions::default(), + VideoSource::Encoded(source) => TrackPublishOptions { + video_codec: source.codec().into(), + video_encoder: VideoEncoderBackend::PreEncoded, + simulcast: false, + ..Default::default() + }, + } + } + + /// Returns a cancellation handle for this pump. + pub fn stop_handle(&self) -> VideoPumpStop { + self.stop.clone() + } + + /// Runs the pump on the calling thread until the source ends, a failure, + /// or the stop handle fires. + /// + /// Sources block, so callers on an async runtime should run this on a + /// dedicated thread (see [`VideoPump::spawn`]) or a blocking pool. + pub fn run(self) -> Result { + match self.source { + VideoSource::Pixel(source) => run_pixel(source, &self.rtc_source, &self.stop), + VideoSource::Encoded(source) => run_encoded(source, &self.rtc_source, &self.stop), + } + } + + /// Runs the pump on a dedicated thread. + /// + /// Panics on the pump thread are caught and reported as + /// [`VideoPumpError::Panicked`] when the pump is joined. + pub fn spawn(self) -> io::Result { + let stop = self.stop_handle(); + let (finished_tx, finished_rx) = tokio::sync::watch::channel(false); + + let thread = thread::Builder::new().name("lk-video-pump".to_owned()).spawn(move || { + let result = catch_unwind(AssertUnwindSafe(|| self.run())) + .unwrap_or_else(|panic| Err(VideoPumpError::Panicked(panic_message(&*panic)))); + let _ = finished_tx.send(true); + result + })?; + + Ok(RunningVideoPump { stop, thread, finished: finished_rx }) + } +} + +/// Renders a panic payload for [`VideoPumpError::Panicked`]. +fn panic_message(panic: &(dyn Any + Send)) -> String { + if let Some(message) = panic.downcast_ref::<&str>() { + (*message).to_owned() + } else if let Some(message) = panic.downcast_ref::() { + message.clone() + } else { + "opaque panic payload".to_owned() + } +} + +/// A [`VideoPump`] running on a dedicated thread. +/// +/// Stopping takes effect between frames: a source blocked waiting for its +/// next frame finishes that wait before the pump observes the signal. +#[derive(Debug)] +pub struct RunningVideoPump { + stop: VideoPumpStop, + thread: thread::JoinHandle>, + /// Flipped to true by the pump thread just before it exits. + finished: tokio::sync::watch::Receiver, +} + +impl RunningVideoPump { + /// Returns a cancellation handle for the pump. + pub fn stop_handle(&self) -> VideoPumpStop { + self.stop.clone() + } + + /// Signals the pump to stop after the frame in flight. + pub fn stop(&self) { + self.stop.stop(); + } + + /// Returns true once the pump thread has exited. + pub fn is_finished(&self) -> bool { + self.thread.is_finished() + } + + /// Waits for the pump thread to exit. + /// + /// Panics on the pump thread are reported as + /// [`VideoPumpError::Panicked`]. + pub fn join(self) -> Result { + self.thread + .join() + .unwrap_or_else(|panic| Err(VideoPumpError::Panicked(panic_message(&*panic)))) + } + + /// Signals the pump to stop and waits for its thread to exit. + pub fn stop_and_join(self) -> Result { + self.stop(); + self.join() + } + + /// Waits for the pump thread to exit without blocking the async runtime. + /// + /// This awaits a completion signal rather than parking a thread, so it is + /// safe to hold across long stretches — for example in a `select!` that + /// supervises every running pump — and works under any async runtime, + /// not just tokio. Panics on the pump thread are reported as + /// [`VideoPumpError::Panicked`]. + pub async fn join_async(mut self) -> Result { + // An error means the sender dropped, which also implies the pump + // thread is done; either way the join below returns promptly. + let _ = self.finished.wait_for(|finished| *finished).await; + self.join() + } + + /// Signals the pump to stop and waits for its thread to exit without + /// blocking the async runtime. + pub async fn stop_and_join_async(self) -> Result { + self.stop(); + self.join_async().await + } +} + +fn run_pixel( + mut source: Box, + rtc_source: &NativeVideoSource, + stop: &VideoPumpStop, +) -> Result { + let mut frames_captured = 0; + let exit = loop { + if stop.is_stopped() { + break VideoPumpExit::Stopped; + } + let Some(frame) = source.next_frame()? else { + break VideoPumpExit::EndOfStream; + }; + capture_pixel_frame(rtc_source, &frame)?; + frames_captured += 1; + }; + Ok(VideoPumpStats { frames_captured, exit }) +} + +fn run_encoded( + mut source: Box, + rtc_source: &NativeVideoSource, + stop: &VideoPumpStop, +) -> Result { + let mut frames_captured = 0; + let mut awaiting_initial_keyframe = true; + let exit = loop { + if stop.is_stopped() { + break VideoPumpExit::Stopped; + } + if let Some(target) = rtc_source.take_rate_control_request() { + source.update_rate_control(target.into()); + } + if rtc_source.take_keyframe_request() { + source.request_keyframe(); + } + + let Some(access_unit) = source.next_access_unit()? else { + break VideoPumpExit::EndOfStream; + }; + + // Drop pre-roll deltas: decoding can only start at a keyframe. + if awaiting_initial_keyframe && access_unit.frame_type != EncodedFrameType::Key { + continue; + } + awaiting_initial_keyframe = false; + + capture_access_unit(rtc_source, &access_unit)?; + frames_captured += 1; + }; + Ok(VideoPumpStats { frames_captured, exit }) +} + +fn capture_pixel_frame( + rtc_source: &NativeVideoSource, + frame: &PixelVideoFrame, +) -> Result<(), CaptureError> { + let buffer = i420_buffer(frame)?; + rtc_source.capture_frame(&VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: frame.timestamp_us, + frame_metadata: None, + buffer, + }); + Ok(()) +} + +fn i420_buffer(frame: &PixelVideoFrame) -> Result { + let PixelVideoData::I420 { y, u, v, stride_y, stride_u, stride_v } = &frame.data; + + let mut buffer = I420Buffer::new(frame.width, frame.height); + let chroma_width = frame.width.div_ceil(2); + let chroma_height = frame.height.div_ceil(2); + let (dst_stride_y, dst_stride_u, dst_stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + + copy_plane(y, *stride_y, dst_y, dst_stride_y, frame.width, frame.height)?; + copy_plane(u, *stride_u, dst_u, dst_stride_u, chroma_width, chroma_height)?; + copy_plane(v, *stride_v, dst_v, dst_stride_v, chroma_width, chroma_height)?; + Ok(buffer) +} + +fn copy_plane( + src: &[u8], + src_stride: u32, + dst: &mut [u8], + dst_stride: u32, + width: u32, + height: u32, +) -> Result<(), CaptureError> { + let (width, height) = (width as usize, height as usize); + let (src_stride, dst_stride) = (src_stride as usize, dst_stride as usize); + if src_stride < width { + return Err(CaptureError::InvalidPixelFrame("plane stride is smaller than its width")); + } + // The final row may be unpadded. + let min_len = (height - 1).saturating_mul(src_stride) + width; + if src.len() < min_len { + return Err(CaptureError::InvalidPixelFrame("plane data is shorter than its dimensions")); + } + + for row in 0..height { + let src_row = &src[row * src_stride..][..width]; + dst[row * dst_stride..][..width].copy_from_slice(src_row); + } + Ok(()) +} + +fn capture_access_unit( + rtc_source: &NativeVideoSource, + access_unit: &OwnedEncodedAccessUnit, +) -> Result<(), CaptureError> { + validate_access_unit(access_unit)?; + + let frame = EncodedVideoFrame { + codec: access_unit.codec.into(), + payload: &access_unit.payload, + timestamp_us: access_unit.timestamp_us, + frame_type: access_unit.frame_type.into(), + resolution: RtcVideoResolution { width: access_unit.width, height: access_unit.height }, + frame_metadata: None, + }; + rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) +} + +/// The passthrough path forwards single-layer streams: access units carrying +/// temporal/spatial layer ids or layering metadata are rejected so callers +/// are not misled into thinking that metadata reaches the wire. +fn validate_access_unit(access_unit: &OwnedEncodedAccessUnit) -> Result<(), CaptureError> { + if access_unit.payload.is_empty() { + return Err(CaptureError::EmptyPayload); + } + if access_unit.layers != EncodedLayerInfo::default() { + return Err(CaptureError::UnsupportedLayeredEncoding( + "temporal/spatial layer ids are not forwarded by the passthrough encoder", + )); + } + if access_unit.codec_specific != CodecSpecific::None + && access_unit.codec_specific != CodecSpecific::default_for(access_unit.codec) + { + return Err(CaptureError::UnsupportedLayeredEncoding( + "codec-specific layering metadata is not forwarded by the passthrough encoder", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use bytes::Bytes; + + use super::*; + use crate::encoded::EncodedVideoCodec; + + const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; + + /// Pixel RTC sources spawn their keepalive task at construction; give the + /// tests the runtime context an SDK application would have. + fn runtime_context() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("failed to build test runtime") + } + + fn pixel_frame(timestamp_us: i64) -> PixelVideoFrame { + let chroma_width = RESOLUTION.width.div_ceil(2); + let chroma_height = RESOLUTION.height.div_ceil(2); + PixelVideoFrame { + width: RESOLUTION.width, + height: RESOLUTION.height, + timestamp_us, + data: PixelVideoData::I420 { + y: Bytes::from(vec![128; (RESOLUTION.width * RESOLUTION.height) as usize]), + u: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), + v: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), + stride_y: RESOLUTION.width, + stride_u: chroma_width, + stride_v: chroma_width, + }, + } + } + + struct FakePixelSource { + frames: VecDeque, + } + + impl FakePixelSource { + fn new(frames: impl IntoIterator) -> Self { + Self { frames: frames.into_iter().collect() } + } + } + + impl PixelVideoSource for FakePixelSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + Ok(self.frames.pop_front()) + } + } + + struct FakeEncodedSource { + access_units: VecDeque, + } + + impl FakeEncodedSource { + fn new(access_units: impl IntoIterator) -> Self { + Self { access_units: access_units.into_iter().collect() } + } + } + + impl EncodedVideoSource for FakeEncodedSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn codec(&self) -> EncodedVideoCodec { + EncodedVideoCodec::VP8 + } + + fn next_access_unit(&mut self) -> Result, SourceError> { + Ok(self.access_units.pop_front()) + } + } + + fn access_unit(timestamp_us: i64, frame_type: EncodedFrameType) -> OwnedEncodedAccessUnit { + OwnedEncodedAccessUnit::new( + EncodedVideoCodec::VP8, + vec![1, 2, 3], + timestamp_us, + frame_type, + RESOLUTION.width, + RESOLUTION.height, + ) + } + + #[test] + fn pixel_pump_captures_all_frames_until_eof() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); + let stats = VideoPump::new(VideoSource::pixel(source)).run().unwrap(); + assert_eq!(stats.frames_captured, 3); + assert_eq!(stats.exit, VideoPumpExit::EndOfStream); + } + + #[test] + fn pump_panics_become_errors() { + struct PanickingSource; + + impl PixelVideoSource for PanickingSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + panic!("source exploded"); + } + } + + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let running = VideoPump::new(VideoSource::pixel(PanickingSource)).spawn().unwrap(); + let error = running.join().unwrap_err(); + assert!( + matches!(&error, VideoPumpError::Panicked(message) if message.contains("source exploded")) + ); + } + + #[test] + fn running_pump_stops_on_signal() { + struct EndlessSource; + + impl PixelVideoSource for EndlessSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok(Some(pixel_frame(0))) + } + } + + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let running = VideoPump::new(VideoSource::pixel(EndlessSource)).spawn().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + let stats = running.stop_and_join().unwrap(); + assert!(stats.frames_captured > 0); + assert_eq!(stats.exit, VideoPumpExit::Stopped); + } + + #[test] + fn pixel_pump_rejects_short_planes() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let mut frame = pixel_frame(1); + let PixelVideoData::I420 { y, .. } = &mut frame.data; + *y = Bytes::from(vec![128; 8]); + + let result = VideoPump::new(VideoSource::pixel(FakePixelSource::new([frame]))).run(); + assert!(matches!(result, Err(VideoPumpError::Capture(CaptureError::InvalidPixelFrame(_))))); + } + + #[tokio::test] + async fn pump_stops_and_joins_async() { + struct EndlessSource; + + impl PixelVideoSource for EndlessSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok(Some(pixel_frame(0))) + } + } + + let running = VideoPump::new(VideoSource::pixel(EndlessSource)).spawn().unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let stats = running.stop_and_join_async().await.unwrap(); + assert!(stats.frames_captured > 0); + } + + #[test] + fn encoded_pump_starts_at_initial_keyframe() { + let source = FakeEncodedSource::new([ + access_unit(1, EncodedFrameType::Delta), + access_unit(2, EncodedFrameType::Delta), + access_unit(3, EncodedFrameType::Key), + access_unit(4, EncodedFrameType::Delta), + ]); + let stats = VideoPump::new(VideoSource::encoded(source)).run().unwrap(); + assert_eq!(stats.frames_captured, 2); + } + + #[test] + fn encoded_pump_rejects_empty_payloads() { + let mut unit = access_unit(1, EncodedFrameType::Key); + unit.payload = Bytes::new(); + + let result = VideoPump::new(VideoSource::encoded(FakeEncodedSource::new([unit]))).run(); + assert!(matches!(result, Err(VideoPumpError::Capture(CaptureError::EmptyPayload)))); + } + + #[test] + fn encoded_publish_options_use_passthrough() { + let pump = VideoPump::new(VideoSource::encoded(FakeEncodedSource::new([]))); + let options = pump.publish_options(); + assert_eq!(options.video_encoder, VideoEncoderBackend::PreEncoded); + assert!(!options.simulcast); + } +} diff --git a/livekit-capture/src/source.rs b/livekit-capture/src/source.rs new file mode 100644 index 000000000..55c674e0b --- /dev/null +++ b/livekit-capture/src/source.rs @@ -0,0 +1,182 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Video capture source traits and types. +//! +//! Everything in this module is independent of libwebrtc: sources produce +//! crate-owned frame types and receive crate-owned feedback types. The +//! [`VideoPump`](crate::pump::VideoPump) bridges a source into an RTC track +//! and mediates all communication with the WebRTC stack. + +use std::{error::Error, fmt}; + +use bytes::Bytes; + +use crate::encoded::{EncodedVideoCodec, OwnedEncodedAccessUnit}; + +/// Video resolution in pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct VideoResolution { + /// Frame width in pixels. + pub width: u32, + /// Frame height in pixels. + pub height: u32, +} + +/// Encoder rate-control target forwarded from WebRTC to an encoded source. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RateControl { + /// Target bitrate in bits per second. + pub target_bitrate_bps: u64, + /// Target frame rate in frames per second. + pub framerate_fps: f64, +} + +/// Error returned by a capture source. +/// +/// Backend-specific errors are type-erased so sources stay usable as trait +/// objects; the wrapped error remains reachable for display and through +/// [`Error::source`]. +#[derive(Debug)] +pub struct SourceError(Box); + +impl SourceError { + /// Wraps a backend error. + pub fn new(error: impl Into>) -> Self { + Self(error.into()) + } +} + +impl fmt::Display for SourceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl Error for SourceError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + self.0.source() + } +} + +/// Pixel data of one video frame. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum PixelVideoData { + /// Planar YUV 4:2:0 with 8-bit samples. + I420 { + /// Luma plane. + y: Bytes, + /// Blue-difference chroma plane. + u: Bytes, + /// Red-difference chroma plane. + v: Bytes, + /// Luma plane stride in bytes. + stride_y: u32, + /// U plane stride in bytes. + stride_u: u32, + /// V plane stride in bytes. + stride_v: u32, + }, +} + +/// One pixel video frame produced by a [`PixelVideoSource`]. +#[derive(Debug, Clone)] +pub struct PixelVideoFrame { + /// Frame width in pixels. + pub width: u32, + /// Frame height in pixels. + pub height: u32, + /// Capture timestamp in microseconds. + pub timestamp_us: i64, + /// Pixel data. + pub data: PixelVideoData, +} + +/// Source of pixel (unencoded) video frames, such as a camera device. +pub trait PixelVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Blocks until the next frame is available, returning `Ok(None)` when + /// the source reaches the end of its stream. + fn next_frame(&mut self) -> Result, SourceError>; +} + +/// Source of pre-encoded video access units, such as an encoding pipeline. +pub trait EncodedVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Codec produced by this source; fixed for the source's lifetime. + fn codec(&self) -> EncodedVideoCodec; + + /// Blocks until the next access unit is available, returning `Ok(None)` + /// when the source reaches the end of its stream. + fn next_access_unit(&mut self) -> Result, SourceError>; + + /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to + /// the producer so it can emit an IDR. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn request_keyframe(&mut self) {} + + /// Forwards a downstream rate-control target to the producer. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn update_rate_control(&mut self, _target: RateControl) {} +} + +/// A video capture source of either kind. +/// +/// This is the unit of dynamic instantiation: backend constructors convert +/// into it, and [`VideoPump::new`](crate::pump::VideoPump::new) consumes it, +/// so applications can build any configured source with one type. +pub enum VideoSource { + /// Source of pixel frames, published through the WebRTC encoder. + Pixel(Box), + /// Source of pre-encoded access units, published as passthrough. + Encoded(Box), +} + +impl VideoSource { + /// Wraps a pixel video source. + pub fn pixel(source: impl PixelVideoSource + 'static) -> Self { + Self::Pixel(Box::new(source)) + } + + /// Wraps an encoded video source. + pub fn encoded(source: impl EncodedVideoSource + 'static) -> Self { + Self::Encoded(Box::new(source)) + } + + /// Nominal output resolution of the underlying source. + pub fn resolution(&self) -> VideoResolution { + match self { + Self::Pixel(source) => source.resolution(), + Self::Encoded(source) => source.resolution(), + } + } +} + +impl fmt::Debug for VideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Pixel(_) => f.debug_tuple("Pixel").finish_non_exhaustive(), + Self::Encoded(_) => f.debug_tuple("Encoded").finish_non_exhaustive(), + } + } +} diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs new file mode 100644 index 000000000..e65294d2a --- /dev/null +++ b/livekit-capture/src/sources/demo.rs @@ -0,0 +1,232 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Solid-color demo source for testing. + +use std::{ + thread, + time::{Duration, Instant}, +}; + +use bytes::Bytes; + +use crate::source::{ + PixelVideoData, PixelVideoFrame, PixelVideoSource, SourceError, VideoResolution, VideoSource, +}; + +/// Colors the demo source cycles through, as `(r, g, b)`. +const PALETTE: [(u8, u8, u8); 6] = [ + (0xE6, 0x32, 0x2E), // red + (0xF4, 0x9D, 0x1A), // orange + (0xF7, 0xD0, 0x38), // yellow + (0x2E, 0xB8, 0x5C), // green + (0x2E, 0x6F, 0xE6), // blue + (0x8E, 0x44, 0xAD), // purple +]; + +/// Configuration for a [`DemoSource`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DemoSourceConfig { + /// Output resolution. + pub resolution: VideoResolution, + /// Output frame rate in frames per second. + pub framerate_fps: u32, + /// How long each palette color is shown before cycling to the next. + pub color_interval: Duration, +} + +impl Default for DemoSourceConfig { + fn default() -> Self { + Self { + resolution: VideoResolution { width: 1280, height: 720 }, + framerate_fps: 30, + color_interval: Duration::from_millis(500), + } + } +} + +/// Pixel video source that produces solid-color frames, cycling through a +/// fixed palette. +/// +/// The source paces itself to the configured frame rate by sleeping and +/// never reaches end of stream; stop the pump driving it instead. It exists +/// to validate capture integration end to end without a device or pipeline +/// dependency. +#[derive(Debug)] +pub struct DemoSource { + config: DemoSourceConfig, + /// One pre-rendered `(y, u, v)` plane set per palette color. + planes: Vec<(Bytes, Bytes, Bytes)>, + started: Option, + frame_index: u64, +} + +impl DemoSource { + /// Creates a demo source. + /// + /// # Panics + /// + /// Panics if the configured resolution or frame rate is zero, or if the + /// color interval is shorter than one frame. + pub fn new(config: DemoSourceConfig) -> Self { + let VideoResolution { width, height } = config.resolution; + assert!(width > 0 && height > 0, "demo source resolution must be non-zero"); + assert!(config.framerate_fps > 0, "demo source frame rate must be non-zero"); + assert!( + config.color_interval >= Duration::from_secs(1) / config.framerate_fps, + "demo source color interval must be at least one frame" + ); + + let luma_len = (width * height) as usize; + let chroma_len = (width.div_ceil(2) * height.div_ceil(2)) as usize; + let planes = PALETTE + .iter() + .map(|&color| { + let (y, u, v) = yuv_from_rgb(color); + ( + Bytes::from(vec![y; luma_len]), + Bytes::from(vec![u; chroma_len]), + Bytes::from(vec![v; chroma_len]), + ) + }) + .collect(); + + Self { config, planes, started: None, frame_index: 0 } + } + + fn frame_interval(&self) -> Duration { + Duration::from_secs(1) / self.config.framerate_fps + } +} + +impl Default for DemoSource { + fn default() -> Self { + Self::new(DemoSourceConfig::default()) + } +} + +impl PixelVideoSource for DemoSource { + fn resolution(&self) -> VideoResolution { + self.config.resolution + } + + fn next_frame(&mut self) -> Result, SourceError> { + let started = *self.started.get_or_insert_with(Instant::now); + + // Pace against the ideal timeline so timestamps stay jitter-free. + let interval_us = self.frame_interval().as_micros() as u64; + let elapsed = Duration::from_micros(self.frame_index.saturating_mul(interval_us)); + let due = started + elapsed; + if let Some(wait) = due.checked_duration_since(Instant::now()) { + thread::sleep(wait); + } + + let timestamp_us = elapsed.as_micros() as i64; + let color_index = + (elapsed.as_micros() / self.config.color_interval.as_micros().max(1)) as usize; + let (y, u, v) = self.planes[color_index % self.planes.len()].clone(); + + self.frame_index += 1; + let VideoResolution { width, height } = self.config.resolution; + Ok(Some(PixelVideoFrame { + width, + height, + timestamp_us, + data: PixelVideoData::I420 { + y, + u, + v, + stride_y: width, + stride_u: width.div_ceil(2), + stride_v: width.div_ceil(2), + }, + })) + } +} + +impl From for VideoSource { + fn from(source: DemoSource) -> Self { + Self::pixel(source) + } +} + +/// Converts an RGB color to limited-range BT.601 YUV. +fn yuv_from_rgb((r, g, b): (u8, u8, u8)) -> (u8, u8, u8) { + let (r, g, b) = (r as f32 / 255.0, g as f32 / 255.0, b as f32 / 255.0); + let y = 16.0 + 65.481 * r + 128.553 * g + 24.966 * b; + let u = 128.0 - 37.797 * r - 74.203 * g + 112.0 * b; + let v = 128.0 + 112.0 * r - 93.786 * g - 18.214 * b; + (y.round() as u8, u.round() as u8, v.round() as u8) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> DemoSourceConfig { + DemoSourceConfig { + resolution: VideoResolution { width: 64, height: 36 }, + framerate_fps: 1000, + color_interval: Duration::from_millis(2), + } + } + + #[test] + fn yields_frames_with_configured_dimensions() { + let mut source = DemoSource::new(test_config()); + + let frame = source.next_frame().unwrap().unwrap(); + assert_eq!((frame.width, frame.height), (64, 36)); + + let PixelVideoData::I420 { y, u, v, .. } = &frame.data; + assert_eq!(y.len(), 64 * 36); + assert_eq!(u.len(), 32 * 18); + assert_eq!(v.len(), 32 * 18); + } + + #[test] + fn timestamps_follow_the_frame_rate() { + let mut source = DemoSource::new(test_config()); + + let first = source.next_frame().unwrap().unwrap(); + let second = source.next_frame().unwrap().unwrap(); + assert_eq!(first.timestamp_us, 0); + assert_eq!(second.timestamp_us, 1_000); + } + + #[test] + fn colors_cycle_at_the_color_interval() { + let mut source = DemoSource::new(test_config()); + + let luma = |frame: &PixelVideoFrame| { + let PixelVideoData::I420 { y, .. } = &frame.data; + y[0] + }; + + // Two frames per color at 1000 fps with a 2 ms interval. + let first = source.next_frame().unwrap().unwrap(); + let same_color = source.next_frame().unwrap().unwrap(); + let next_color = source.next_frame().unwrap().unwrap(); + assert_eq!(luma(&first), luma(&same_color)); + assert_ne!(luma(&first), luma(&next_color)); + } + + #[test] + fn converts_primaries_to_expected_luma() { + // White has maximum luma and centered chroma in limited range. + assert_eq!(yuv_from_rgb((255, 255, 255)), (235, 128, 128)); + // Black has minimum luma and centered chroma. + assert_eq!(yuv_from_rgb((0, 0, 0)), (16, 128, 128)); + } +} diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index 307f844d1..f5529ea7b 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -14,5 +14,8 @@ //! Optional capture sources that feed the shared capture paths. +#[cfg(feature = "demo")] +pub mod demo; + #[cfg(feature = "gstreamer")] pub mod gstreamer; From 58486cd48a46e4d3d49d058bc5e1e502c2f8468d Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:54:21 -0700 Subject: [PATCH 16/56] Separate pump types --- livekit-capture/README.md | 20 +- livekit-capture/src/lib.rs | 4 +- livekit-capture/src/pump.rs | 399 +++++++++++++++++----------- livekit-capture/src/source.rs | 68 ++--- livekit-capture/src/sources/demo.rs | 8 +- 5 files changed, 303 insertions(+), 196 deletions(-) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index d912f5176..a1fc07fa0 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -1,11 +1,25 @@ # livekit-capture -Helpers for publishing pre-encoded video with the LiveKit Rust SDK. The -optional `gstreamer` feature turns a GStreamer `appsink` into an encoded -ingest source. +Capture sources and helpers for publishing video with the LiveKit Rust SDK. +The optional `gstreamer` feature turns a GStreamer `appsink` into an encoded +ingest source; the `demo` feature adds a synthetic pixel source for testing. ## Library entry points +- `source::PixelVideoSource` and `source::EncodedVideoSource` — the + libwebrtc-free traits a capture backend implements: pixel sources produce + frames published through the WebRTC encoder, encoded sources produce + access units published as passthrough. Both traits are object-safe and + implemented for `Box`, so sources can be constructed dynamically + and driven through the same pumps. +- `pump::PixelPump` and `pump::EncodedPump` — bridge a source into a + publishable RTC track: each builds the matching `NativeVideoSource`, + derives publish options (`EncodedPump` selects the passthrough encoder), + and runs the capture loop on a plain thread. Encoded pumps forward + downstream keyframe and rate-control requests back to the source and drop + pre-roll deltas until the first keyframe. Both spawn into the same + `pump::RunningPump`, so an application supervises running pumps of either + kind uniformly (`stop()`, `join_async()`, stats). - `track::NativeVideoSourceExt` — extension methods on the RTC-level `NativeVideoSource` for capturing pre-encoded access units. Use `NativeVideoSource::new_encoded` for pre-encoded passthrough (no raw diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index fcfef51f4..7af066789 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -22,11 +22,11 @@ pub mod sources; pub mod track; pub use pump::{ - RunningVideoPump, VideoPump, VideoPumpError, VideoPumpExit, VideoPumpStats, VideoPumpStop, + EncodedPump, PixelPump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump, }; pub use source::{ EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, - SourceError, VideoResolution, VideoSource, + SourceError, VideoResolution, }; #[cfg(feature = "demo")] pub use sources::demo::{DemoSource, DemoSourceConfig}; diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index a61f01810..df2283c9b 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -14,15 +14,22 @@ //! Pumps frames from a capture source into an RTC video source. //! -//! [`VideoPump`] is the bridge between the libwebrtc-free source traits in -//! [`source`](crate::source) and a publishable RTC track: it builds the -//! matching [`NativeVideoSource`], converts crate-owned frame types at the -//! boundary, and forwards downstream keyframe and rate-control requests back -//! to encoded sources. +//! [`PixelPump`] and [`EncodedPump`] are the bridges between the +//! libwebrtc-free source traits in [`source`](crate::source) and a +//! publishable RTC track: each builds the matching [`NativeVideoSource`], +//! converts crate-owned frame types at the boundary, and — for encoded +//! sources — forwards downstream keyframe and rate-control requests back to +//! the producer. +//! +//! Both pumps are generic over a concrete source, so statically-known +//! sources pay for no type erasure. Applications that construct sources +//! dynamically box them at their edge (`PixelPump>`); both pumps spawn into the same [`RunningPump`], so +//! running pumps of either kind are handled uniformly. use std::{ any::Any, - io, + fmt, io, panic::{catch_unwind, AssertUnwindSafe}, sync::{ atomic::{AtomicBool, Ordering}, @@ -48,7 +55,7 @@ use crate::{ error::CaptureError, source::{ EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, - SourceError, VideoResolution, VideoSource, + SourceError, VideoResolution, }, }; @@ -66,7 +73,7 @@ impl From for RtcVideoResolution { /// Error returned by a pump run. #[derive(Debug, Error)] -pub enum VideoPumpError { +pub enum PumpError { /// The capture source failed. #[error("capture source failed")] Source(#[from] SourceError), @@ -80,7 +87,7 @@ pub enum VideoPumpError { /// Why a pump run ended successfully. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VideoPumpExit { +pub enum PumpExit { /// The stop handle was fired. Stopped, /// The source reached the end of its stream. @@ -90,22 +97,21 @@ pub enum VideoPumpExit { /// Statistics returned when a pump run ends. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] -pub struct VideoPumpStats { +pub struct PumpStats { /// Number of frames or access units captured. pub frames_captured: u64, /// Why the run ended. - pub exit: VideoPumpExit, + pub exit: PumpExit, } -/// Cancellation handle for a [`VideoPump`]. +/// Cancellation handle for a pump. /// -/// Cheap to clone; wire it to a shutdown signal and call -/// [`VideoPumpStop::stop`] from any thread to make the pump return after the -/// frame in flight. +/// Cheap to clone; wire it to a shutdown signal and call [`PumpStop::stop`] +/// from any thread to make the pump return after the frame in flight. #[derive(Debug, Clone, Default)] -pub struct VideoPumpStop(Arc); +pub struct PumpStop(Arc); -impl VideoPumpStop { +impl PumpStop { /// Creates an un-stopped handle. pub fn new() -> Self { Self::default() @@ -116,40 +122,117 @@ impl VideoPumpStop { self.0.store(true, Ordering::Release); } - /// Returns true once [`VideoPumpStop::stop`] has been called. + /// Returns true once [`PumpStop::stop`] has been called. pub fn is_stopped(&self) -> bool { self.0.load(Ordering::Acquire) } } -/// Pumps a [`VideoSource`] into an RTC video source. -/// -/// The pump owns every libwebrtc interaction: it builds the RTC source -/// appropriate for the capture source kind, derives the matching publish -/// options, converts frames at the boundary, and polls downstream keyframe -/// and rate-control requests, forwarding them to encoded sources as -/// crate-owned types. -#[derive(Debug)] -pub struct VideoPump { - source: VideoSource, +/// Pumps a [`PixelVideoSource`] into an RTC video source, publishing frames +/// through the WebRTC encoder. +pub struct PixelPump { + source: S, rtc_source: NativeVideoSource, - stop: VideoPumpStop, + stop: PumpStop, } -impl VideoPump { - /// Creates a pump for a capture source, building the matching RTC source. +impl PixelPump { + /// Creates a pump for a pixel source, building the matching RTC source. /// - /// For pixel sources this must be called from the context of the async - /// runtime driving the SDK, because the RTC source spawns its keepalive - /// task at construction. The pump itself runs on plain threads. - pub fn new(source: impl Into) -> Self { - let source = source.into(); - let resolution = source.resolution().into(); - let rtc_source = match &source { - VideoSource::Pixel(_) => NativeVideoSource::new(resolution, false), - VideoSource::Encoded(_) => NativeVideoSource::new_encoded(resolution), + /// This must be called from the context of the async runtime driving the + /// SDK, because the RTC source spawns its keepalive task at construction. + /// The pump itself runs on plain threads. + pub fn new(source: S) -> Self { + let rtc_source = NativeVideoSource::new(source.resolution().into(), false); + Self { source, rtc_source, stop: PumpStop::new() } + } + + /// Returns the RTC source to create the local track with. + pub fn rtc_source(&self) -> RtcVideoSource { + RtcVideoSource::Native(self.rtc_source.clone()) + } + + /// Returns publish options appropriate for a pixel source. + pub fn publish_options(&self) -> TrackPublishOptions { + TrackPublishOptions::default() + } + + /// Returns a cancellation handle for this pump. + pub fn stop_handle(&self) -> PumpStop { + self.stop.clone() + } + + /// Returns the underlying capture source. + pub fn source(&self) -> &S { + &self.source + } + + /// Returns the underlying capture source mutably. + pub fn source_mut(&mut self) -> &mut S { + &mut self.source + } + + /// Runs the pump on the calling thread until the source ends, a failure, + /// or the stop handle fires. + /// + /// Sources block, so callers on an async runtime should run this on a + /// dedicated thread (see [`PixelPump::spawn`]) or a blocking pool. + pub fn run(mut self) -> Result { + let mut frames_captured = 0; + let exit = loop { + if self.stop.is_stopped() { + break PumpExit::Stopped; + } + let Some(frame) = self.source.next_frame()? else { + break PumpExit::EndOfStream; + }; + capture_pixel_frame(&self.rtc_source, &frame)?; + frames_captured += 1; }; - Self { source, rtc_source, stop: VideoPumpStop::new() } + Ok(PumpStats { frames_captured, exit }) + } + + /// Runs the pump on a dedicated thread. + /// + /// Panics on the pump thread are caught and reported as + /// [`PumpError::Panicked`] when the pump is joined. + pub fn spawn(self) -> io::Result + where + S: 'static, + { + let stop = self.stop_handle(); + spawn_pump(stop, move || self.run()) + } +} + +impl fmt::Debug for PixelPump { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PixelPump") + .field("rtc_source", &self.rtc_source) + .field("stop", &self.stop) + .finish_non_exhaustive() + } +} + +/// Pumps an [`EncodedVideoSource`] into an RTC video source, publishing +/// access units as passthrough. +/// +/// Downstream keyframe requests (PLI/FIR, late subscriber) and rate-control +/// targets are polled between access units and forwarded to the source. +/// Pre-roll delta frames are dropped until the first keyframe, since +/// decoding can only start at a keyframe. +pub struct EncodedPump { + source: S, + rtc_source: NativeVideoSource, + stop: PumpStop, +} + +impl EncodedPump { + /// Creates a pump for an encoded source, building the matching RTC + /// source. + pub fn new(source: S) -> Self { + let rtc_source = NativeVideoSource::new_encoded(source.resolution().into()); + Self { source, rtc_source, stop: PumpStop::new() } } /// Returns the RTC source to create the local track with. @@ -157,56 +240,107 @@ impl VideoPump { RtcVideoSource::Native(self.rtc_source.clone()) } - /// Returns publish options appropriate for the source kind. + /// Returns publish options for encoded passthrough. pub fn publish_options(&self) -> TrackPublishOptions { - match &self.source { - VideoSource::Pixel(_) => TrackPublishOptions::default(), - VideoSource::Encoded(source) => TrackPublishOptions { - video_codec: source.codec().into(), - video_encoder: VideoEncoderBackend::PreEncoded, - simulcast: false, - ..Default::default() - }, + TrackPublishOptions { + video_codec: self.source.codec().into(), + video_encoder: VideoEncoderBackend::PreEncoded, + simulcast: false, + ..Default::default() } } /// Returns a cancellation handle for this pump. - pub fn stop_handle(&self) -> VideoPumpStop { + pub fn stop_handle(&self) -> PumpStop { self.stop.clone() } + /// Returns the underlying capture source. + pub fn source(&self) -> &S { + &self.source + } + + /// Returns the underlying capture source mutably. + pub fn source_mut(&mut self) -> &mut S { + &mut self.source + } + /// Runs the pump on the calling thread until the source ends, a failure, /// or the stop handle fires. /// /// Sources block, so callers on an async runtime should run this on a - /// dedicated thread (see [`VideoPump::spawn`]) or a blocking pool. - pub fn run(self) -> Result { - match self.source { - VideoSource::Pixel(source) => run_pixel(source, &self.rtc_source, &self.stop), - VideoSource::Encoded(source) => run_encoded(source, &self.rtc_source, &self.stop), - } + /// dedicated thread (see [`EncodedPump::spawn`]) or a blocking pool. + pub fn run(mut self) -> Result { + let mut frames_captured = 0; + let mut awaiting_initial_keyframe = true; + let exit = loop { + if self.stop.is_stopped() { + break PumpExit::Stopped; + } + if let Some(target) = self.rtc_source.take_rate_control_request() { + self.source.update_rate_control(target.into()); + } + if self.rtc_source.take_keyframe_request() { + self.source.request_keyframe(); + } + + let Some(access_unit) = self.source.next_access_unit()? else { + break PumpExit::EndOfStream; + }; + + // Drop pre-roll deltas: decoding can only start at a keyframe. + if awaiting_initial_keyframe && access_unit.frame_type != EncodedFrameType::Key { + continue; + } + awaiting_initial_keyframe = false; + + capture_access_unit(&self.rtc_source, &access_unit)?; + frames_captured += 1; + }; + Ok(PumpStats { frames_captured, exit }) } /// Runs the pump on a dedicated thread. /// /// Panics on the pump thread are caught and reported as - /// [`VideoPumpError::Panicked`] when the pump is joined. - pub fn spawn(self) -> io::Result { + /// [`PumpError::Panicked`] when the pump is joined. + pub fn spawn(self) -> io::Result + where + S: 'static, + { let stop = self.stop_handle(); - let (finished_tx, finished_rx) = tokio::sync::watch::channel(false); - - let thread = thread::Builder::new().name("lk-video-pump".to_owned()).spawn(move || { - let result = catch_unwind(AssertUnwindSafe(|| self.run())) - .unwrap_or_else(|panic| Err(VideoPumpError::Panicked(panic_message(&*panic)))); - let _ = finished_tx.send(true); - result - })?; + spawn_pump(stop, move || self.run()) + } +} - Ok(RunningVideoPump { stop, thread, finished: finished_rx }) +impl fmt::Debug for EncodedPump { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EncodedPump") + .field("rtc_source", &self.rtc_source) + .field("stop", &self.stop) + .finish_non_exhaustive() } } -/// Renders a panic payload for [`VideoPumpError::Panicked`]. +/// Spawns a pump run on a dedicated thread, wiring panic capture and the +/// completion signal shared by both pump kinds. +fn spawn_pump( + stop: PumpStop, + run: impl FnOnce() -> Result + Send + 'static, +) -> io::Result { + let (finished_tx, finished_rx) = tokio::sync::watch::channel(false); + + let thread = thread::Builder::new().name("lk-video-pump".to_owned()).spawn(move || { + let result = catch_unwind(AssertUnwindSafe(run)) + .unwrap_or_else(|panic| Err(PumpError::Panicked(panic_message(&*panic)))); + let _ = finished_tx.send(true); + result + })?; + + Ok(RunningPump { stop, thread, finished: finished_rx }) +} + +/// Renders a panic payload for [`PumpError::Panicked`]. fn panic_message(panic: &(dyn Any + Send)) -> String { if let Some(message) = panic.downcast_ref::<&str>() { (*message).to_owned() @@ -217,21 +351,21 @@ fn panic_message(panic: &(dyn Any + Send)) -> String { } } -/// A [`VideoPump`] running on a dedicated thread. +/// A pump of either kind running on a dedicated thread. /// /// Stopping takes effect between frames: a source blocked waiting for its /// next frame finishes that wait before the pump observes the signal. #[derive(Debug)] -pub struct RunningVideoPump { - stop: VideoPumpStop, - thread: thread::JoinHandle>, +pub struct RunningPump { + stop: PumpStop, + thread: thread::JoinHandle>, /// Flipped to true by the pump thread just before it exits. finished: tokio::sync::watch::Receiver, } -impl RunningVideoPump { +impl RunningPump { /// Returns a cancellation handle for the pump. - pub fn stop_handle(&self) -> VideoPumpStop { + pub fn stop_handle(&self) -> PumpStop { self.stop.clone() } @@ -247,16 +381,13 @@ impl RunningVideoPump { /// Waits for the pump thread to exit. /// - /// Panics on the pump thread are reported as - /// [`VideoPumpError::Panicked`]. - pub fn join(self) -> Result { - self.thread - .join() - .unwrap_or_else(|panic| Err(VideoPumpError::Panicked(panic_message(&*panic)))) + /// Panics on the pump thread are reported as [`PumpError::Panicked`]. + pub fn join(self) -> Result { + self.thread.join().unwrap_or_else(|panic| Err(PumpError::Panicked(panic_message(&*panic)))) } /// Signals the pump to stop and waits for its thread to exit. - pub fn stop_and_join(self) -> Result { + pub fn stop_and_join(self) -> Result { self.stop(); self.join() } @@ -267,8 +398,8 @@ impl RunningVideoPump { /// safe to hold across long stretches — for example in a `select!` that /// supervises every running pump — and works under any async runtime, /// not just tokio. Panics on the pump thread are reported as - /// [`VideoPumpError::Panicked`]. - pub async fn join_async(mut self) -> Result { + /// [`PumpError::Panicked`]. + pub async fn join_async(mut self) -> Result { // An error means the sender dropped, which also implies the pump // thread is done; either way the join below returns promptly. let _ = self.finished.wait_for(|finished| *finished).await; @@ -277,65 +408,12 @@ impl RunningVideoPump { /// Signals the pump to stop and waits for its thread to exit without /// blocking the async runtime. - pub async fn stop_and_join_async(self) -> Result { + pub async fn stop_and_join_async(self) -> Result { self.stop(); self.join_async().await } } -fn run_pixel( - mut source: Box, - rtc_source: &NativeVideoSource, - stop: &VideoPumpStop, -) -> Result { - let mut frames_captured = 0; - let exit = loop { - if stop.is_stopped() { - break VideoPumpExit::Stopped; - } - let Some(frame) = source.next_frame()? else { - break VideoPumpExit::EndOfStream; - }; - capture_pixel_frame(rtc_source, &frame)?; - frames_captured += 1; - }; - Ok(VideoPumpStats { frames_captured, exit }) -} - -fn run_encoded( - mut source: Box, - rtc_source: &NativeVideoSource, - stop: &VideoPumpStop, -) -> Result { - let mut frames_captured = 0; - let mut awaiting_initial_keyframe = true; - let exit = loop { - if stop.is_stopped() { - break VideoPumpExit::Stopped; - } - if let Some(target) = rtc_source.take_rate_control_request() { - source.update_rate_control(target.into()); - } - if rtc_source.take_keyframe_request() { - source.request_keyframe(); - } - - let Some(access_unit) = source.next_access_unit()? else { - break VideoPumpExit::EndOfStream; - }; - - // Drop pre-roll deltas: decoding can only start at a keyframe. - if awaiting_initial_keyframe && access_unit.frame_type != EncodedFrameType::Key { - continue; - } - awaiting_initial_keyframe = false; - - capture_access_unit(rtc_source, &access_unit)?; - frames_captured += 1; - }; - Ok(VideoPumpStats { frames_captured, exit }) -} - fn capture_pixel_frame( rtc_source: &NativeVideoSource, frame: &PixelVideoFrame, @@ -529,9 +607,28 @@ mod tests { let _guard = runtime.enter(); let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); - let stats = VideoPump::new(VideoSource::pixel(source)).run().unwrap(); + let stats = PixelPump::new(source).run().unwrap(); assert_eq!(stats.frames_captured, 3); - assert_eq!(stats.exit, VideoPumpExit::EndOfStream); + assert_eq!(stats.exit, PumpExit::EndOfStream); + } + + #[test] + fn boxed_sources_drive_generic_pumps() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + // The dynamic-instantiation pattern: box at the edge, same pumps. + let source: Box = + Box::new(FakePixelSource::new([pixel_frame(1), pixel_frame(2)])); + let stats = PixelPump::new(source).run().unwrap(); + assert_eq!(stats.frames_captured, 2); + + let source: Box = + Box::new(FakeEncodedSource::new([access_unit(1, EncodedFrameType::Key)])); + let pump = EncodedPump::new(source); + assert_eq!(pump.publish_options().video_encoder, VideoEncoderBackend::PreEncoded); + let stats = pump.run().unwrap(); + assert_eq!(stats.frames_captured, 1); } #[test] @@ -551,10 +648,10 @@ mod tests { let runtime = runtime_context(); let _guard = runtime.enter(); - let running = VideoPump::new(VideoSource::pixel(PanickingSource)).spawn().unwrap(); + let running = PixelPump::new(PanickingSource).spawn().unwrap(); let error = running.join().unwrap_err(); assert!( - matches!(&error, VideoPumpError::Panicked(message) if message.contains("source exploded")) + matches!(&error, PumpError::Panicked(message) if message.contains("source exploded")) ); } @@ -576,11 +673,11 @@ mod tests { let runtime = runtime_context(); let _guard = runtime.enter(); - let running = VideoPump::new(VideoSource::pixel(EndlessSource)).spawn().unwrap(); + let running = PixelPump::new(EndlessSource).spawn().unwrap(); std::thread::sleep(std::time::Duration::from_millis(20)); let stats = running.stop_and_join().unwrap(); assert!(stats.frames_captured > 0); - assert_eq!(stats.exit, VideoPumpExit::Stopped); + assert_eq!(stats.exit, PumpExit::Stopped); } #[test] @@ -592,8 +689,8 @@ mod tests { let PixelVideoData::I420 { y, .. } = &mut frame.data; *y = Bytes::from(vec![128; 8]); - let result = VideoPump::new(VideoSource::pixel(FakePixelSource::new([frame]))).run(); - assert!(matches!(result, Err(VideoPumpError::Capture(CaptureError::InvalidPixelFrame(_))))); + let result = PixelPump::new(FakePixelSource::new([frame])).run(); + assert!(matches!(result, Err(PumpError::Capture(CaptureError::InvalidPixelFrame(_))))); } #[tokio::test] @@ -611,7 +708,7 @@ mod tests { } } - let running = VideoPump::new(VideoSource::pixel(EndlessSource)).spawn().unwrap(); + let running = PixelPump::new(EndlessSource).spawn().unwrap(); tokio::time::sleep(std::time::Duration::from_millis(20)).await; let stats = running.stop_and_join_async().await.unwrap(); assert!(stats.frames_captured > 0); @@ -625,7 +722,7 @@ mod tests { access_unit(3, EncodedFrameType::Key), access_unit(4, EncodedFrameType::Delta), ]); - let stats = VideoPump::new(VideoSource::encoded(source)).run().unwrap(); + let stats = EncodedPump::new(source).run().unwrap(); assert_eq!(stats.frames_captured, 2); } @@ -634,13 +731,13 @@ mod tests { let mut unit = access_unit(1, EncodedFrameType::Key); unit.payload = Bytes::new(); - let result = VideoPump::new(VideoSource::encoded(FakeEncodedSource::new([unit]))).run(); - assert!(matches!(result, Err(VideoPumpError::Capture(CaptureError::EmptyPayload)))); + let result = EncodedPump::new(FakeEncodedSource::new([unit])).run(); + assert!(matches!(result, Err(PumpError::Capture(CaptureError::EmptyPayload)))); } #[test] fn encoded_publish_options_use_passthrough() { - let pump = VideoPump::new(VideoSource::encoded(FakeEncodedSource::new([]))); + let pump = EncodedPump::new(FakeEncodedSource::new([])); let options = pump.publish_options(); assert_eq!(options.video_encoder, VideoEncoderBackend::PreEncoded); assert!(!options.simulcast); diff --git a/livekit-capture/src/source.rs b/livekit-capture/src/source.rs index 55c674e0b..ffbab1f42 100644 --- a/livekit-capture/src/source.rs +++ b/livekit-capture/src/source.rs @@ -15,9 +15,14 @@ //! Video capture source traits and types. //! //! Everything in this module is independent of libwebrtc: sources produce -//! crate-owned frame types and receive crate-owned feedback types. The -//! [`VideoPump`](crate::pump::VideoPump) bridges a source into an RTC track -//! and mediates all communication with the WebRTC stack. +//! crate-owned frame types and receive crate-owned feedback types. The pumps +//! in [`pump`](crate::pump) bridge a source into an RTC track and mediate all +//! communication with the WebRTC stack. +//! +//! Both source traits are object-safe, and `Box` boxes implement +//! them, so applications that construct sources dynamically can drive a +//! [`PixelPump>`](crate::pump::PixelPump) while +//! applications that know their source statically pay for no type erasure. use std::{error::Error, fmt}; @@ -140,43 +145,40 @@ pub trait EncodedVideoSource: Send { fn update_rate_control(&mut self, _target: RateControl) {} } -/// A video capture source of either kind. -/// -/// This is the unit of dynamic instantiation: backend constructors convert -/// into it, and [`VideoPump::new`](crate::pump::VideoPump::new) consumes it, -/// so applications can build any configured source with one type. -pub enum VideoSource { - /// Source of pixel frames, published through the WebRTC encoder. - Pixel(Box), - /// Source of pre-encoded access units, published as passthrough. - Encoded(Box), +impl PixelVideoSource for Box { + fn resolution(&self) -> VideoResolution { + (**self).resolution() + } + + fn next_frame(&mut self) -> Result, SourceError> { + (**self).next_frame() + } } -impl VideoSource { - /// Wraps a pixel video source. - pub fn pixel(source: impl PixelVideoSource + 'static) -> Self { - Self::Pixel(Box::new(source)) +impl EncodedVideoSource for Box { + fn resolution(&self) -> VideoResolution { + (**self).resolution() } - /// Wraps an encoded video source. - pub fn encoded(source: impl EncodedVideoSource + 'static) -> Self { - Self::Encoded(Box::new(source)) + fn codec(&self) -> EncodedVideoCodec { + (**self).codec() } - /// Nominal output resolution of the underlying source. - pub fn resolution(&self) -> VideoResolution { - match self { - Self::Pixel(source) => source.resolution(), - Self::Encoded(source) => source.resolution(), - } + fn next_access_unit(&mut self) -> Result, SourceError> { + (**self).next_access_unit() } -} -impl fmt::Debug for VideoSource { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Pixel(_) => f.debug_tuple("Pixel").finish_non_exhaustive(), - Self::Encoded(_) => f.debug_tuple("Encoded").finish_non_exhaustive(), - } + fn request_keyframe(&mut self) { + (**self).request_keyframe() + } + + fn update_rate_control(&mut self, target: RateControl) { + (**self).update_rate_control(target) } } + +// Object safety is part of these traits' contract: dynamic applications box +// sources at their edge and drive them through the same generic pumps. +const _: () = { + fn _assert_object_safe(_: &dyn PixelVideoSource, _: &dyn EncodedVideoSource) {} +}; diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index e65294d2a..fd7cc9e1f 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -22,7 +22,7 @@ use std::{ use bytes::Bytes; use crate::source::{ - PixelVideoData, PixelVideoFrame, PixelVideoSource, SourceError, VideoResolution, VideoSource, + PixelVideoData, PixelVideoFrame, PixelVideoSource, SourceError, VideoResolution, }; /// Colors the demo source cycles through, as `(r, g, b)`. @@ -155,12 +155,6 @@ impl PixelVideoSource for DemoSource { } } -impl From for VideoSource { - fn from(source: DemoSource) -> Self { - Self::pixel(source) - } -} - /// Converts an RGB color to limited-range BT.601 YUV. fn yuv_from_rgb((r, g, b): (u8, u8, u8)) -> (u8, u8, u8) { let (r, g, b) = (r as f32 / 255.0, g as f32 / 255.0, b as f32 / 255.0); From fdca59011cb262760ab4286116556f79ef587f8f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:17:33 -0700 Subject: [PATCH 17/56] Rename pumps to be video specific --- livekit-capture/README.md | 4 ++-- livekit-capture/src/lib.rs | 2 +- livekit-capture/src/pump.rs | 44 +++++++++++++++++------------------ livekit-capture/src/source.rs | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index a1fc07fa0..8db573310 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -12,9 +12,9 @@ ingest source; the `demo` feature adds a synthetic pixel source for testing. access units published as passthrough. Both traits are object-safe and implemented for `Box`, so sources can be constructed dynamically and driven through the same pumps. -- `pump::PixelPump` and `pump::EncodedPump` — bridge a source into a +- `pump::PixelVideoPump` and `pump::EncodedVideoPump` — bridge a source into a publishable RTC track: each builds the matching `NativeVideoSource`, - derives publish options (`EncodedPump` selects the passthrough encoder), + derives publish options (`EncodedVideoPump` selects the passthrough encoder), and runs the capture loop on a plain thread. Encoded pumps forward downstream keyframe and rate-control requests back to the source and drop pre-roll deltas until the first keyframe. Both spawn into the same diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 7af066789..56c68dd55 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -22,7 +22,7 @@ pub mod sources; pub mod track; pub use pump::{ - EncodedPump, PixelPump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump, + EncodedVideoPump, PixelVideoPump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump, }; pub use source::{ EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index df2283c9b..bd779b714 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -14,7 +14,7 @@ //! Pumps frames from a capture source into an RTC video source. //! -//! [`PixelPump`] and [`EncodedPump`] are the bridges between the +//! [`PixelVideoPump`] and [`EncodedVideoPump`] are the bridges between the //! libwebrtc-free source traits in [`source`](crate::source) and a //! publishable RTC track: each builds the matching [`NativeVideoSource`], //! converts crate-owned frame types at the boundary, and — for encoded @@ -23,7 +23,7 @@ //! //! Both pumps are generic over a concrete source, so statically-known //! sources pay for no type erasure. Applications that construct sources -//! dynamically box them at their edge (`PixelPump>`); both pumps spawn into the same [`RunningPump`], so //! running pumps of either kind are handled uniformly. @@ -130,13 +130,13 @@ impl PumpStop { /// Pumps a [`PixelVideoSource`] into an RTC video source, publishing frames /// through the WebRTC encoder. -pub struct PixelPump { +pub struct PixelVideoPump { source: S, rtc_source: NativeVideoSource, stop: PumpStop, } -impl PixelPump { +impl PixelVideoPump { /// Creates a pump for a pixel source, building the matching RTC source. /// /// This must be called from the context of the async runtime driving the @@ -176,7 +176,7 @@ impl PixelPump { /// or the stop handle fires. /// /// Sources block, so callers on an async runtime should run this on a - /// dedicated thread (see [`PixelPump::spawn`]) or a blocking pool. + /// dedicated thread (see [`PixelVideoPump::spawn`]) or a blocking pool. pub fn run(mut self) -> Result { let mut frames_captured = 0; let exit = loop { @@ -205,9 +205,9 @@ impl PixelPump { } } -impl fmt::Debug for PixelPump { +impl fmt::Debug for PixelVideoPump { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PixelPump") + f.debug_struct("PixelVideoPump") .field("rtc_source", &self.rtc_source) .field("stop", &self.stop) .finish_non_exhaustive() @@ -221,13 +221,13 @@ impl fmt::Debug for PixelPump { /// targets are polled between access units and forwarded to the source. /// Pre-roll delta frames are dropped until the first keyframe, since /// decoding can only start at a keyframe. -pub struct EncodedPump { +pub struct EncodedVideoPump { source: S, rtc_source: NativeVideoSource, stop: PumpStop, } -impl EncodedPump { +impl EncodedVideoPump { /// Creates a pump for an encoded source, building the matching RTC /// source. pub fn new(source: S) -> Self { @@ -269,7 +269,7 @@ impl EncodedPump { /// or the stop handle fires. /// /// Sources block, so callers on an async runtime should run this on a - /// dedicated thread (see [`EncodedPump::spawn`]) or a blocking pool. + /// dedicated thread (see [`EncodedVideoPump::spawn`]) or a blocking pool. pub fn run(mut self) -> Result { let mut frames_captured = 0; let mut awaiting_initial_keyframe = true; @@ -313,9 +313,9 @@ impl EncodedPump { } } -impl fmt::Debug for EncodedPump { +impl fmt::Debug for EncodedVideoPump { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("EncodedPump") + f.debug_struct("EncodedVideoPump") .field("rtc_source", &self.rtc_source) .field("stop", &self.stop) .finish_non_exhaustive() @@ -607,7 +607,7 @@ mod tests { let _guard = runtime.enter(); let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); - let stats = PixelPump::new(source).run().unwrap(); + let stats = PixelVideoPump::new(source).run().unwrap(); assert_eq!(stats.frames_captured, 3); assert_eq!(stats.exit, PumpExit::EndOfStream); } @@ -620,12 +620,12 @@ mod tests { // The dynamic-instantiation pattern: box at the edge, same pumps. let source: Box = Box::new(FakePixelSource::new([pixel_frame(1), pixel_frame(2)])); - let stats = PixelPump::new(source).run().unwrap(); + let stats = PixelVideoPump::new(source).run().unwrap(); assert_eq!(stats.frames_captured, 2); let source: Box = Box::new(FakeEncodedSource::new([access_unit(1, EncodedFrameType::Key)])); - let pump = EncodedPump::new(source); + let pump = EncodedVideoPump::new(source); assert_eq!(pump.publish_options().video_encoder, VideoEncoderBackend::PreEncoded); let stats = pump.run().unwrap(); assert_eq!(stats.frames_captured, 1); @@ -648,7 +648,7 @@ mod tests { let runtime = runtime_context(); let _guard = runtime.enter(); - let running = PixelPump::new(PanickingSource).spawn().unwrap(); + let running = PixelVideoPump::new(PanickingSource).spawn().unwrap(); let error = running.join().unwrap_err(); assert!( matches!(&error, PumpError::Panicked(message) if message.contains("source exploded")) @@ -673,7 +673,7 @@ mod tests { let runtime = runtime_context(); let _guard = runtime.enter(); - let running = PixelPump::new(EndlessSource).spawn().unwrap(); + let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); std::thread::sleep(std::time::Duration::from_millis(20)); let stats = running.stop_and_join().unwrap(); assert!(stats.frames_captured > 0); @@ -689,7 +689,7 @@ mod tests { let PixelVideoData::I420 { y, .. } = &mut frame.data; *y = Bytes::from(vec![128; 8]); - let result = PixelPump::new(FakePixelSource::new([frame])).run(); + let result = PixelVideoPump::new(FakePixelSource::new([frame])).run(); assert!(matches!(result, Err(PumpError::Capture(CaptureError::InvalidPixelFrame(_))))); } @@ -708,7 +708,7 @@ mod tests { } } - let running = PixelPump::new(EndlessSource).spawn().unwrap(); + let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); tokio::time::sleep(std::time::Duration::from_millis(20)).await; let stats = running.stop_and_join_async().await.unwrap(); assert!(stats.frames_captured > 0); @@ -722,7 +722,7 @@ mod tests { access_unit(3, EncodedFrameType::Key), access_unit(4, EncodedFrameType::Delta), ]); - let stats = EncodedPump::new(source).run().unwrap(); + let stats = EncodedVideoPump::new(source).run().unwrap(); assert_eq!(stats.frames_captured, 2); } @@ -731,13 +731,13 @@ mod tests { let mut unit = access_unit(1, EncodedFrameType::Key); unit.payload = Bytes::new(); - let result = EncodedPump::new(FakeEncodedSource::new([unit])).run(); + let result = EncodedVideoPump::new(FakeEncodedSource::new([unit])).run(); assert!(matches!(result, Err(PumpError::Capture(CaptureError::EmptyPayload)))); } #[test] fn encoded_publish_options_use_passthrough() { - let pump = EncodedPump::new(FakeEncodedSource::new([])); + let pump = EncodedVideoPump::new(FakeEncodedSource::new([])); let options = pump.publish_options(); assert_eq!(options.video_encoder, VideoEncoderBackend::PreEncoded); assert!(!options.simulcast); diff --git a/livekit-capture/src/source.rs b/livekit-capture/src/source.rs index ffbab1f42..6f0ce0078 100644 --- a/livekit-capture/src/source.rs +++ b/livekit-capture/src/source.rs @@ -21,7 +21,7 @@ //! //! Both source traits are object-safe, and `Box` boxes implement //! them, so applications that construct sources dynamically can drive a -//! [`PixelPump>`](crate::pump::PixelPump) while +//! [`PixelVideoPump>`](crate::pump::PixelVideoPump) while //! applications that know their source statically pay for no type erasure. use std::{error::Error, fmt}; From 6390e592a990eb49331ed5bed7935e22ec8ecbbf Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:29:06 -0700 Subject: [PATCH 18/56] Update gstreamer source to use new interface --- livekit-capture/README.md | 22 +- livekit-capture/src/encoded/ingress.rs | 319 ----------------------- livekit-capture/src/encoded/mod.rs | 4 - livekit-capture/src/lib.rs | 9 +- livekit-capture/src/sources/gstreamer.rs | 108 +++----- livekit-capture/src/track.rs | 200 -------------- 6 files changed, 42 insertions(+), 620 deletions(-) delete mode 100644 livekit-capture/src/encoded/ingress.rs delete mode 100644 livekit-capture/src/track.rs diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 8db573310..b3128fc59 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -20,16 +20,6 @@ ingest source; the `demo` feature adds a synthetic pixel source for testing. pre-roll deltas until the first keyframe. Both spawn into the same `pump::RunningPump`, so an application supervises running pumps of either kind uniformly (`stop()`, `join_async()`, stats). -- `track::NativeVideoSourceExt` — extension methods on the RTC-level - `NativeVideoSource` for capturing pre-encoded access units. Use - `NativeVideoSource::new_encoded` for pre-encoded passthrough (no raw - keepalive frames, so the sender starts directly on the passthrough encoder). -- `EncodedIngress` — the pre-encoded pump used when the caller manages its - own source: `capture_next()` reports each published access unit, - `stop_handle()` cancels from any thread, and downstream keyframe requests - (PLI/FIR) are forwarded to the source automatically. Passthrough is - single-layer (`L1T1`), and access units carrying other layering metadata are - rejected. - `sources::gstreamer::ensure_encoded_appsink` and friends turn an arbitrary pipeline (containing `appsink name=lk_appsink` or one unlinked encoded pad) into an encoded source; `encoded_caps_string` is the single per-codec caps @@ -38,9 +28,9 @@ ingest source; the `demo` feature adds a synthetic pixel source for testing. ## GStreamer ingest -`GStreamerAppSinkEncodedSource` implements `EncodedAccessUnitSource` on top of -an `appsink` producing H.264 (Annex-B or AVC), H.265 Annex-B, VP8, VP9, or -AV1 access units. Feed it to `EncodedIngress` together with a -`NativeVideoSource::new_encoded` RTC source, then publish a local video track -created from that source with `track::encoded_publish_options(codec)` so the -sender uses the pre-encoded passthrough encoder. +`GStreamerVideoSource` implements `EncodedVideoSource` on top of an +`appsink` producing H.264 (Annex-B or AVC), H.265 Annex-B, VP8, VP9, or AV1 +access units. Drive it with an `EncodedVideoPump`, which builds the encoded +RTC source, derives the passthrough publish options, and forwards keyframe +and rate-control requests back to the pipeline. Passthrough is single-layer +(`L1T1`); access units carrying other layering metadata are rejected. diff --git a/livekit-capture/src/encoded/ingress.rs b/livekit-capture/src/encoded/ingress.rs deleted file mode 100644 index 1513bd5ce..000000000 --- a/livekit-capture/src/encoded/ingress.rs +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{ - error::Error, - fmt, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; - -use livekit::webrtc::{video_frame::FrameMetadata, video_source::native::NativeVideoSource}; - -use crate::{ - encoded::{EncodedFrameType, EncodedRateControl, OwnedEncodedAccessUnit}, - error::CaptureError, - track::NativeVideoSourceExt, -}; - -/// Source of owned encoded access units. -pub trait EncodedAccessUnitSource { - /// Error returned by the source. - type Error: Error + Send + Sync + 'static; - - /// Returns the next encoded access unit, or `Ok(None)` when the source reaches EOF. - fn next_access_unit(&mut self) -> Result, Self::Error>; - - /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to - /// the producer so it can emit an IDR. - /// - /// The default implementation does nothing, for transports that cannot - /// influence the upstream encoder. - fn request_keyframe(&mut self) {} - - /// Forwards a downstream rate-control target to the producer. - /// - /// The default implementation does nothing, for transports that cannot - /// influence the upstream encoder. - fn update_rate_control(&mut self, _rate_control: EncodedRateControl) {} -} - -/// Error returned while forwarding encoded access units into a track. -#[derive(Debug)] -pub enum EncodedIngressError { - /// The encoded source failed. - Source(E), - /// The capture track rejected an access unit. - Capture(CaptureError), -} - -impl fmt::Display for EncodedIngressError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Source(err) => write!(f, "encoded source failed: {err}"), - Self::Capture(err) => write!(f, "encoded capture failed: {err}"), - } - } -} - -impl Error for EncodedIngressError -where - E: Error + 'static, -{ - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Source(err) => Some(err), - Self::Capture(err) => Some(err), - } - } -} - -/// Cancellation handle for [`EncodedIngress::run_until_end`]. -/// -/// Cheap to clone; wire it to a shutdown signal (e.g. Ctrl-C) and call -/// [`EncodedIngressStop::stop`] from any thread to make the ingest loop -/// return after the access unit in flight. -#[derive(Debug, Clone, Default)] -pub struct EncodedIngressStop(Arc); - -impl EncodedIngressStop { - /// Creates an un-stopped handle. - pub fn new() -> Self { - Self::default() - } - - /// Signals the ingest loop to stop. - pub fn stop(&self) { - self.0.store(true, Ordering::Release); - } - - /// Returns true once [`EncodedIngressStop::stop`] has been called. - pub fn is_stopped(&self) -> bool { - self.0.load(Ordering::Acquire) - } -} - -/// Pulls encoded access units from a source and forwards them into a video track. -#[derive(Debug)] -pub struct EncodedIngress { - rtc_source: NativeVideoSource, - capture_source: S, - stop: EncodedIngressStop, - awaiting_initial_keyframe: bool, -} - -impl EncodedIngress { - /// Creates an encoded ingress runner. - pub fn new(rtc_source: NativeVideoSource, capture_source: S) -> Self { - Self { - rtc_source, - capture_source, - stop: EncodedIngressStop::new(), - awaiting_initial_keyframe: true, - } - } - - /// Returns a cancellation handle for this runner. - pub fn stop_handle(&self) -> EncodedIngressStop { - self.stop.clone() - } - - /// Returns the RTC source used by this runner. - pub fn rtc_source(&self) -> &NativeVideoSource { - &self.rtc_source - } - - /// Returns the underlying encoded source. - pub fn source(&self) -> &S { - &self.capture_source - } - - /// Returns the underlying encoded source mutably. - pub fn source_mut(&mut self) -> &mut S { - &mut self.capture_source - } - - /// Consumes this runner and returns its parts. - pub fn into_parts(self) -> (NativeVideoSource, S) { - (self.rtc_source, self.capture_source) - } -} - -/// Details of one access unit captured by [`EncodedIngress::capture_next`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct EncodedIngressCapture { - /// Capture timestamp of the access unit in microseconds. - pub timestamp_us: i64, - /// Frame type of the access unit. - pub frame_type: crate::encoded::EncodedFrameType, - /// Payload size in bytes. - pub payload_len: usize, -} - -impl EncodedIngress -where - S: EncodedAccessUnitSource, -{ - /// Captures the next access unit, returning `None` after source EOF. - /// - /// Downstream rate-control and keyframe requests raised by the - /// passthrough encoder are polled on every call and forwarded to the - /// source via [`EncodedAccessUnitSource::update_rate_control`] and - /// [`EncodedAccessUnitSource::request_keyframe`]. - pub fn capture_next( - &mut self, - ) -> Result, EncodedIngressError> { - self.capture_next_with_metadata(|_| None) - } - - /// Captures the next access unit with metadata generated after the source yields it. - /// - /// The metadata producer is not called for skipped pre-roll frames while - /// the ingress runner is waiting for the initial keyframe. - pub fn capture_next_with_metadata( - &mut self, - frame_metadata: impl FnOnce(&OwnedEncodedAccessUnit) -> Option, - ) -> Result, EncodedIngressError> { - if let Some(rate_control) = self.rtc_source.take_rate_control_request() { - self.capture_source.update_rate_control(rate_control); - } - if self.rtc_source.take_keyframe_request() { - self.capture_source.request_keyframe(); - } - - let access_unit = loop { - let Some(access_unit) = - self.capture_source.next_access_unit().map_err(EncodedIngressError::Source)? - else { - return Ok(None); - }; - - if !self.awaiting_initial_keyframe || access_unit.frame_type == EncodedFrameType::Key { - self.awaiting_initial_keyframe = false; - break access_unit; - } - }; - - let frame_metadata = frame_metadata(&access_unit); - self.rtc_source - .capture_encoded_with_metadata(&access_unit.as_access_unit(), frame_metadata) - .map_err(EncodedIngressError::Capture)?; - Ok(Some(EncodedIngressCapture { - timestamp_us: access_unit.timestamp_us, - frame_type: access_unit.frame_type, - payload_len: access_unit.payload.len(), - })) - } - - /// Captures access units until the source reaches EOF or the stop - /// handle fires, returning the number of captured access units. - pub fn run_until_end(&mut self) -> Result> { - let mut captured = 0; - while !self.stop.is_stopped() && self.capture_next()?.is_some() { - captured += 1; - } - Ok(captured) - } -} - -#[cfg(test)] -mod tests { - use std::{collections::VecDeque, error::Error, fmt}; - - use livekit::webrtc::video_source::VideoResolution; - - use super::*; - use crate::encoded::EncodedVideoCodec; - - #[derive(Debug)] - struct FakeSourceError; - - impl fmt::Display for FakeSourceError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("fake source failed") - } - } - - impl Error for FakeSourceError {} - - #[derive(Debug)] - struct FakeSource { - access_units: VecDeque, - } - - impl FakeSource { - fn new(access_units: impl IntoIterator) -> Self { - Self { access_units: access_units.into_iter().collect() } - } - } - - impl EncodedAccessUnitSource for FakeSource { - type Error = FakeSourceError; - - fn next_access_unit(&mut self) -> Result, Self::Error> { - Ok(self.access_units.pop_front()) - } - } - - fn access_unit(timestamp_us: i64, frame_type: EncodedFrameType) -> OwnedEncodedAccessUnit { - OwnedEncodedAccessUnit::new( - EncodedVideoCodec::VP8, - vec![1, 2, 3], - timestamp_us, - frame_type, - 640, - 480, - ) - } - - fn rtc_source() -> NativeVideoSource { - NativeVideoSource::new_encoded(VideoResolution { width: 640, height: 480 }) - } - - #[test] - fn capture_next_starts_at_initial_keyframe() { - let source = FakeSource::new([ - access_unit(1, EncodedFrameType::Delta), - access_unit(2, EncodedFrameType::Delta), - access_unit(3, EncodedFrameType::Key), - ]); - let mut ingress = EncodedIngress::new(rtc_source(), source); - - let capture = ingress - .capture_next() - .expect("capture should succeed") - .expect("keyframe should be captured"); - - assert_eq!(capture.timestamp_us, 3); - assert_eq!(capture.frame_type, EncodedFrameType::Key); - } - - #[test] - fn capture_next_allows_deltas_after_initial_keyframe() { - let source = FakeSource::new([ - access_unit(1, EncodedFrameType::Key), - access_unit(2, EncodedFrameType::Delta), - ]); - let mut ingress = EncodedIngress::new(rtc_source(), source); - - let first = ingress.capture_next().unwrap().unwrap(); - let second = ingress.capture_next().unwrap().unwrap(); - - assert_eq!(first.frame_type, EncodedFrameType::Key); - assert_eq!(second.frame_type, EncodedFrameType::Delta); - assert_eq!(second.timestamp_us, 2); - } -} diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index ed889c6cf..41787d8b1 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. pub mod h26x; -pub mod ingress; use bytes::Bytes; use livekit::{ @@ -27,9 +26,6 @@ use crate::error::CaptureError; const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; -/// Encoder rate-control target requested by WebRTC for an encoded source. -pub use livekit::webrtc::video_source::EncodedRateControl; - /// Encoded byte-stream framing used by encoded source backends. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 56c68dd55..6f64eec61 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -19,7 +19,6 @@ mod error; pub mod pump; pub mod source; pub mod sources; -pub mod track; pub use pump::{ EncodedVideoPump, PixelVideoPump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump, @@ -32,12 +31,8 @@ pub use source::{ pub use sources::demo::{DemoSource, DemoSourceConfig}; pub use encoded::{ - ingress::{ - EncodedAccessUnitSource, EncodedIngress, EncodedIngressCapture, EncodedIngressError, - EncodedIngressStop, - }, CodecSpecific, EncodedAccessUnit, EncodedFragment, EncodedFrameType, EncodedLayerInfo, - EncodedPayload, EncodedRateControl, EncodedVideoCodec, EncodedWireFormat, - H264PacketizationMode, OwnedEncodedAccessUnit, + EncodedPayload, EncodedVideoCodec, EncodedWireFormat, H264PacketizationMode, + OwnedEncodedAccessUnit, }; pub use error::CaptureError; diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 3f5f93296..4664cca7a 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::error::Error as StdError; - use bytes::Bytes; use thiserror::Error; @@ -25,11 +23,10 @@ use gst::prelude::*; use crate::{ encoded::{ h26x::{access_unit_from_annex_b, access_unit_from_h264_avc}, - ingress::EncodedAccessUnitSource, - CodecSpecific, EncodedFrameType, EncodedRateControl, EncodedVideoCodec, - OwnedEncodedAccessUnit, + CodecSpecific, EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, }, error::CaptureError, + source::{EncodedVideoSource, RateControl, SourceError, VideoResolution}, }; /// Encoded sample format expected from a GStreamer appsink. @@ -66,29 +63,26 @@ impl GStreamerSampleFormat { /// Configuration for a GStreamer appsink encoded source. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct GStreamerAppSinkConfig { +pub struct GStreamerVideoSourceConfig { /// Format of encoded buffers pulled from appsink. pub sample_format: GStreamerSampleFormat, /// Timestamp added to the first buffer timestamp, or used directly as fallback. pub start_timestamp_us: i64, /// Fallback frame interval when a GStreamer buffer has no PTS or DTS. pub frame_interval_us: i64, - /// Encoded frame width in pixels. - pub width: u32, - /// Encoded frame height in pixels. - pub height: u32, + /// Encoded frame resolution in pixels. + pub resolution: VideoResolution, } -impl GStreamerAppSinkConfig { +impl GStreamerVideoSourceConfig { /// Creates GStreamer appsink source configuration. pub fn new( sample_format: GStreamerSampleFormat, start_timestamp_us: i64, frame_interval_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Self { - Self { sample_format, start_timestamp_us, frame_interval_us, width, height } + Self { sample_format, start_timestamp_us, frame_interval_us, resolution } } } @@ -110,7 +104,7 @@ impl GStreamerBitrateUnit { } } -/// GStreamer encoder bitrate control used by [`GStreamerAppSinkEncodedSource`]. +/// GStreamer encoder bitrate control used by [`GStreamerVideoSource`]. #[derive(Debug, Clone)] pub struct GStreamerEncoderRateControl { encoder: gst::Element, @@ -134,7 +128,7 @@ impl GStreamerEncoderRateControl { } } - fn update(&mut self, rate_control: EncodedRateControl) { + fn update(&mut self, rate_control: RateControl) { if self.last_target_bitrate_bps == Some(rate_control.target_bitrate_bps) { return; } @@ -156,16 +150,16 @@ impl GStreamerEncoderRateControl { /// Encoded source backed by a GStreamer appsink. #[derive(Debug)] -pub struct GStreamerAppSinkEncodedSource { +pub struct GStreamerVideoSource { appsink: gst_app::AppSink, - config: GStreamerAppSinkConfig, + config: GStreamerVideoSourceConfig, next_fallback_timestamp_us: i64, rate_control: Option, } -impl GStreamerAppSinkEncodedSource { +impl GStreamerVideoSource { /// Creates an encoded source from an existing GStreamer appsink. - pub fn new(appsink: gst_app::AppSink, config: GStreamerAppSinkConfig) -> Self { + pub fn new(appsink: gst_app::AppSink, config: GStreamerVideoSourceConfig) -> Self { Self { appsink, config, @@ -185,7 +179,7 @@ impl GStreamerAppSinkEncodedSource { } /// Returns the source configuration. - pub fn config(&self) -> GStreamerAppSinkConfig { + pub fn config(&self) -> GStreamerVideoSourceConfig { self.config } @@ -197,8 +191,8 @@ impl GStreamerAppSinkEncodedSource { fn access_unit_from_sample( &mut self, sample: &gst::Sample, - ) -> Result { - let buffer = sample.buffer().ok_or(GStreamerSourceError::MissingBuffer)?; + ) -> Result { + let buffer = sample.buffer().ok_or(GStreamerVideoSourceError::MissingBuffer)?; let timestamp_us = self.timestamp_us(buffer); let frame_type = if buffer.flags().contains(gst::BufferFlags::DELTA_UNIT) { EncodedFrameType::Delta @@ -208,17 +202,17 @@ impl GStreamerAppSinkEncodedSource { let map = buffer .map_readable() - .map_err(|err| GStreamerSourceError::MapReadable(err.to_string()))?; + .map_err(|err| GStreamerVideoSourceError::MapReadable(err.to_string()))?; let payload = map.as_ref(); access_unit_from_sample_payload( self.config.sample_format, payload, timestamp_us, frame_type, - self.config.width, - self.config.height, + self.config.resolution.width, + self.config.resolution.height, ) - .map_err(GStreamerSourceError::Capture) + .map_err(GStreamerVideoSourceError::Capture) } fn timestamp_us(&mut self, buffer: &gst::BufferRef) -> i64 { @@ -237,14 +231,20 @@ impl GStreamerAppSinkEncodedSource { } } -impl EncodedAccessUnitSource for GStreamerAppSinkEncodedSource { - type Error = GStreamerSourceError; +impl EncodedVideoSource for GStreamerVideoSource { + fn resolution(&self) -> VideoResolution { + self.config.resolution + } - fn next_access_unit(&mut self) -> Result, Self::Error> { + fn codec(&self) -> EncodedVideoCodec { + self.config.sample_format.codec() + } + + fn next_access_unit(&mut self) -> Result, SourceError> { match self.appsink.pull_sample() { - Ok(sample) => self.access_unit_from_sample(&sample).map(Some), + Ok(sample) => self.access_unit_from_sample(&sample).map(Some).map_err(SourceError::new), Err(_err) if self.appsink.is_eos() => Ok(None), - Err(err) => Err(GStreamerSourceError::PullSample(err.to_string())), + Err(err) => Err(SourceError::new(GStreamerVideoSourceError::PullSample(err.to_string()))), } } @@ -257,7 +257,7 @@ impl EncodedAccessUnitSource for GStreamerAppSinkEncodedSource { let _ = self.appsink.send_event(gst::event::CustomUpstream::new(structure)); } - fn update_rate_control(&mut self, rate_control: EncodedRateControl) { + fn update_rate_control(&mut self, rate_control: RateControl) { if let Some(control) = &mut self.rate_control { control.update(rate_control); } @@ -316,7 +316,7 @@ fn clamp_to_i64(value: u64, minimum: i64, maximum: i64) -> i64 { /// Error returned by GStreamer appsink encoded sources. #[derive(Debug, Error)] -pub enum GStreamerSourceError { +pub enum GStreamerVideoSourceError { /// The appsink failed to produce a sample. #[error("failed to pull GStreamer appsink sample: {0}")] PullSample(String), @@ -331,46 +331,6 @@ pub enum GStreamerSourceError { Capture(CaptureError), } -/// Callback-backed encoded source for GStreamer appsink integrations. -#[derive(Debug)] -pub struct GStreamerAppSinkSource { - next_access_unit: F, -} - -impl GStreamerAppSinkSource { - /// Creates a source from a callback that pulls the next encoded appsink sample. - pub fn new(next_access_unit: F) -> Self { - Self { next_access_unit } - } - - /// Returns the wrapped callback. - pub fn callback(&self) -> &F { - &self.next_access_unit - } - - /// Returns the wrapped callback mutably. - pub fn callback_mut(&mut self) -> &mut F { - &mut self.next_access_unit - } - - /// Consumes this source and returns the wrapped callback. - pub fn into_callback(self) -> F { - self.next_access_unit - } -} - -impl EncodedAccessUnitSource for GStreamerAppSinkSource -where - F: FnMut() -> Result, E>, - E: StdError + Send + Sync + 'static, -{ - type Error = E; - - fn next_access_unit(&mut self) -> Result, Self::Error> { - (self.next_access_unit)() - } -} - fn access_unit_from_sample_payload( sample_format: GStreamerSampleFormat, payload: &[u8], diff --git a/livekit-capture/src/track.rs b/livekit-capture/src/track.rs deleted file mode 100644 index ab41f5352..000000000 --- a/livekit-capture/src/track.rs +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use livekit::{ - options::{TrackPublishOptions, VideoEncoderBackend}, - webrtc::{ - video_frame::{EncodedVideoFrame, FrameMetadata}, - video_source::{native::NativeVideoSource, VideoResolution}, - }, -}; - -use crate::{ - encoded::{ - CodecSpecific, EncodedAccessUnit, EncodedLayerInfo, EncodedPayload, EncodedVideoCodec, - }, - error::CaptureError, -}; - -/// Additional methods for [`NativeVideoSource`] to support capture from sources. -pub trait NativeVideoSourceExt { - /// Captures one encoded video access unit. - /// - /// The passthrough path forwards single-layer streams: access units - /// carrying temporal/spatial layer ids, an AV1 dependency descriptor, or - /// a non-`L1T1` scalability mode are rejected so callers are not misled - /// into thinking that metadata reaches the wire. - fn capture_encoded(&self, access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError>; - - /// Captures one encoded video access unit with optional frame metadata. - /// - /// Metadata is only propagated to subscribers when the corresponding - /// [`TrackPublishOptions::frame_metadata_features`] are enabled before - /// publishing the local track. - fn capture_encoded_with_metadata( - &self, - access_unit: &EncodedAccessUnit<'_>, - frame_metadata: Option, - ) -> Result<(), CaptureError>; -} - -impl NativeVideoSourceExt for NativeVideoSource { - fn capture_encoded(&self, access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError> { - self.capture_encoded_with_metadata(access_unit, None) - } - - fn capture_encoded_with_metadata( - &self, - access_unit: &EncodedAccessUnit<'_>, - frame_metadata: Option, - ) -> Result<(), CaptureError> { - validate_encoded_access_unit(access_unit)?; - - let scratch; - let payload: &[u8] = match &access_unit.payload { - EncodedPayload::Contiguous(bytes) => bytes, - EncodedPayload::Owned(bytes) => bytes, - EncodedPayload::Fragments(_) => { - scratch = access_unit.payload.to_vec(); - &scratch - } - }; - let frame = EncodedVideoFrame { - codec: access_unit.codec.into(), - payload, - timestamp_us: access_unit.timestamp_us, - frame_type: access_unit.frame_type.into(), - resolution: VideoResolution { width: access_unit.width, height: access_unit.height }, - frame_metadata, - }; - self.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) - } -} - -/// Returns publish options appropriate for encoded passthrough. -pub fn encoded_publish_options(codec: EncodedVideoCodec) -> TrackPublishOptions { - TrackPublishOptions { - video_codec: codec.into(), - video_encoder: VideoEncoderBackend::PreEncoded, - simulcast: false, - ..Default::default() - } -} - -fn validate_encoded_access_unit(access_unit: &EncodedAccessUnit<'_>) -> Result<(), CaptureError> { - if access_unit.payload.is_empty() { - return Err(CaptureError::EmptyPayload); - } - if access_unit.layers != EncodedLayerInfo::default() { - return Err(CaptureError::UnsupportedLayeredEncoding( - "temporal/spatial layer ids are not forwarded by the passthrough encoder", - )); - } - let default_specific = CodecSpecific::default_for(access_unit.codec); - if access_unit.codec_specific != CodecSpecific::None - && access_unit.codec_specific != default_specific - { - return Err(CaptureError::UnsupportedLayeredEncoding( - "codec-specific layering metadata is not forwarded by the passthrough encoder", - )); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::encoded::EncodedFrameType; - - #[test] - fn accepts_vp8_vp9_and_av1_access_units() { - for codec in [EncodedVideoCodec::VP8, EncodedVideoCodec::VP9, EncodedVideoCodec::AV1] { - let access_unit = EncodedAccessUnit::contiguous( - codec, - &[1, 2, 3], - 0, - EncodedFrameType::Key, - 640, - 480, - ); - - assert!(validate_encoded_access_unit(&access_unit).is_ok()); - } - } - - #[test] - fn rejects_empty_encoded_access_units() { - let access_unit = EncodedAccessUnit::contiguous( - EncodedVideoCodec::VP8, - &[], - 0, - EncodedFrameType::Key, - 640, - 480, - ); - - assert_eq!(validate_encoded_access_unit(&access_unit), Err(CaptureError::EmptyPayload)); - } - - #[test] - fn accepts_default_codec_specific_metadata() { - let mut access_unit = EncodedAccessUnit::contiguous( - EncodedVideoCodec::AV1, - &[1, 2, 3], - 0, - EncodedFrameType::Key, - 640, - 480, - ); - access_unit.codec_specific = CodecSpecific::default_for(EncodedVideoCodec::AV1); - - assert!(validate_encoded_access_unit(&access_unit).is_ok()); - } - - #[test] - fn rejects_layered_access_units() { - let mut access_unit = EncodedAccessUnit::contiguous( - EncodedVideoCodec::VP9, - &[1, 2, 3], - 0, - EncodedFrameType::Key, - 640, - 480, - ); - access_unit.layers = EncodedLayerInfo { spatial_id: None, temporal_id: Some(1) }; - - assert!(matches!( - validate_encoded_access_unit(&access_unit), - Err(CaptureError::UnsupportedLayeredEncoding(_)) - )); - } - - #[test] - fn rejects_non_default_codec_specific_metadata() { - let mut access_unit = EncodedAccessUnit::contiguous( - EncodedVideoCodec::VP8, - &[1, 2, 3], - 0, - EncodedFrameType::Key, - 640, - 480, - ); - access_unit.codec_specific = CodecSpecific::VP8 { temporal_id: Some(1), layer_sync: true }; - - assert!(matches!( - validate_encoded_access_unit(&access_unit), - Err(CaptureError::UnsupportedLayeredEncoding(_)) - )); - } -} From 80748f4a85493b080b0cc59acdc1cd589c8f28cd Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:38:30 -0700 Subject: [PATCH 19/56] Add video resolution primitive --- livekit-capture/src/encoded/h26x.rs | 72 ++++++++++-------------- livekit-capture/src/encoded/mod.rs | 61 ++++++++------------ livekit-capture/src/lib.rs | 4 +- livekit-capture/src/primitive.rs | 69 +++++++++++++++++++++++ livekit-capture/src/pump.rs | 30 ++++------ livekit-capture/src/source.rs | 20 ++----- livekit-capture/src/sources/demo.rs | 12 ++-- livekit-capture/src/sources/gstreamer.rs | 35 +++++------- 8 files changed, 162 insertions(+), 141 deletions(-) create mode 100644 livekit-capture/src/primitive.rs diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index d33567655..baa44d017 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -22,6 +22,7 @@ use crate::{ EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, }, error::CaptureError, + primitive::VideoResolution, }; /// Upper bound on bytes buffered while waiting for an access-unit boundary. @@ -58,8 +59,7 @@ pub struct AnnexBAccessUnitParser { scan_cursor: usize, next_timestamp_us: i64, frame_interval_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, } /// H.264/AVC length-prefixed parser state. @@ -74,8 +74,7 @@ pub(crate) struct AvcAccessUnitParser { nal_length_size: u8, next_timestamp_us: i64, frame_interval_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, } impl AnnexBAccessUnitParser { @@ -84,8 +83,7 @@ impl AnnexBAccessUnitParser { codec: EncodedVideoCodec, start_timestamp_us: i64, frame_interval_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result { match codec { EncodedVideoCodec::H264 | EncodedVideoCodec::H265 => {} @@ -101,8 +99,7 @@ impl AnnexBAccessUnitParser { scan_cursor: 0, next_timestamp_us: start_timestamp_us, frame_interval_us, - width, - height, + resolution, }) } @@ -188,8 +185,7 @@ impl AnnexBAccessUnitParser { self.codec, Bytes::from(access_unit), timestamp_us, - self.width, - self.height, + self.resolution, ) .map(Some) } @@ -213,8 +209,7 @@ impl AvcAccessUnitParser { nal_length_size: u8, start_timestamp_us: i64, frame_interval_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result { validate_avc_nal_length_size(nal_length_size)?; @@ -225,8 +220,7 @@ impl AvcAccessUnitParser { nal_length_size, next_timestamp_us: start_timestamp_us, frame_interval_us, - width, - height, + resolution, }) } @@ -323,8 +317,7 @@ impl AvcAccessUnitParser { &access_unit, self.nal_length_size, timestamp_us, - self.width, - self.height, + self.resolution, ) .map(Some) } @@ -382,11 +375,10 @@ pub fn access_unit_from_h264_avc( payload: &[u8], nal_length_size: u8, timestamp_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result { let nals = avc_nalus(payload, nal_length_size)?; - access_unit_from_nalus(EncodedVideoCodec::H264, &nals, timestamp_us, width, height) + access_unit_from_nalus(EncodedVideoCodec::H264, &nals, timestamp_us, resolution) } /// Creates an access unit from an Annex-B buffer. @@ -394,8 +386,7 @@ pub fn access_unit_from_annex_b( codec: EncodedVideoCodec, payload: Bytes, timestamp_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result { if payload.is_empty() { return Err(CaptureError::EmptyPayload); @@ -407,7 +398,7 @@ pub fn access_unit_from_annex_b( EncodedFrameType::Delta }; let mut access_unit = - OwnedEncodedAccessUnit::new(codec, payload, timestamp_us, frame_type, width, height); + OwnedEncodedAccessUnit::new(codec, payload, timestamp_us, frame_type, resolution); access_unit.codec_specific = CodecSpecific::default_for(codec); Ok(access_unit) } @@ -417,11 +408,10 @@ pub fn access_unit_from_nalus( codec: EncodedVideoCodec, nal_units: &[&[u8]], timestamp_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result { let payload = Bytes::from(annex_b_payload(nal_units)?); - access_unit_from_annex_b(codec, payload, timestamp_us, width, height) + access_unit_from_annex_b(codec, payload, timestamp_us, resolution) } /// Returns true when an Annex-B access unit contains an intra/key picture. @@ -629,7 +619,7 @@ mod tests { #[test] fn access_unit_from_avc_converts_length_prefixed_nals() { let bytes = [0, 0, 0, 4, 0x67, 1, 2, 3, 0, 0, 0, 3, 0x65, 4, 5]; - let au = access_unit_from_h264_avc(&bytes, 4, 10, 640, 480).unwrap(); + let au = access_unit_from_h264_avc(&bytes, 4, 10, VideoResolution::new(640, 480)).unwrap(); assert_eq!(au.codec, EncodedVideoCodec::H264); assert_eq!(au.frame_type, EncodedFrameType::Key); @@ -639,7 +629,7 @@ mod tests { #[test] fn access_unit_from_avc_supports_two_byte_lengths() { let bytes = [0, 2, 0x61, 1]; - let au = access_unit_from_h264_avc(&bytes, 2, 10, 640, 480).unwrap(); + let au = access_unit_from_h264_avc(&bytes, 2, 10, VideoResolution::new(640, 480)).unwrap(); assert_eq!(au.frame_type, EncodedFrameType::Delta); assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x61, 1]); @@ -647,7 +637,7 @@ mod tests { #[test] fn access_unit_from_avc_rejects_truncated_nal() { - let err = access_unit_from_h264_avc(&[0, 0, 0, 3, 0x65], 4, 10, 640, 480).unwrap_err(); + let err = access_unit_from_h264_avc(&[0, 0, 0, 3, 0x65], 4, 10, VideoResolution::new(640, 480)).unwrap_err(); assert_eq!(err, CaptureError::InvalidEncodedData("truncated AVC NAL unit")); } @@ -655,7 +645,7 @@ mod tests { #[test] fn parser_flushes_final_access_unit() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); assert!(parser.push(&[0, 0, 1, 0x65, 1, 2]).unwrap().is_none()); let au = parser.flush().unwrap().unwrap(); assert_eq!(au.timestamp_us, 100); @@ -665,7 +655,7 @@ mod tests { #[test] fn parser_splits_at_next_access_unit_delimiter() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x65, 1, 2, 0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x41, 3]; @@ -680,7 +670,7 @@ mod tests { #[test] fn avc_parser_splits_at_next_access_unit_delimiter() { - let mut parser = AvcAccessUnitParser::new(4, 100, 33_333, 640, 480).unwrap(); + let mut parser = AvcAccessUnitParser::new(4, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [ 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 3, 0x65, 1, 2, 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 2, 0x41, 3, @@ -698,7 +688,7 @@ mod tests { #[test] fn splits_aud_less_h264_stream_per_frame() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [ 0, 0, 0, 1, 0x67, 0x42, 0x00, 0x1e, // SPS 0, 0, 0, 1, 0x68, 0xce, // PPS @@ -725,7 +715,7 @@ mod tests { #[test] fn keeps_multi_slice_h264_access_unit_together() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [ 0, 0, 1, 0x65, 0x88, 0x11, // IDR slice, first_mb_in_slice == 0 0, 0, 1, 0x65, 0x21, 0x22, // IDR slice, first_mb_in_slice != 0 @@ -745,7 +735,7 @@ mod tests { #[test] fn splits_aud_less_h265_stream_per_frame() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [ 0, 0, 0, 1, 0x40, 0x01, 0x0c, // VPS 0, 0, 0, 1, 0x42, 0x01, 0x02, // SPS @@ -769,7 +759,7 @@ mod tests { #[test] fn keeps_multi_slice_h265_access_unit_together() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [ 0, 0, 1, 0x26, 0x01, 0xaf, 0x11, // IDR slice, first_slice_segment_in_pic_flag == 1 @@ -791,7 +781,7 @@ mod tests { #[test] fn groups_parameter_sets_with_following_frame() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [ 0, 0, 1, 0x67, 0x42, 0x1e, // SPS 0, 0, 1, 0x68, 0xce, // PPS @@ -857,7 +847,7 @@ mod tests { 0, 0, 0, 1, 0x41, 0x9a, 0x04, 0x00, // P, first_mb_in_slice == 0 ]; assert_chunked_matches_one_shot( - || AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(), + || AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(), &h264_annex_b, 4, ); @@ -872,7 +862,7 @@ mod tests { 0, 0, 1, 0x02, 0x01, 0xd0, 0x0a, // TRAIL_R ]; assert_chunked_matches_one_shot( - || AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, 640, 480).unwrap(), + || AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, VideoResolution::new(640, 480)).unwrap(), &h265_annex_b, 3, ); @@ -886,7 +876,7 @@ mod tests { 0, 0, 0, 3, 0x41, 0x9a, 0x03, // P ]; assert_chunked_matches_one_shot( - || AvcAccessUnitParser::new(4, 0, 33_333, 640, 480).unwrap(), + || AvcAccessUnitParser::new(4, 0, 33_333, VideoResolution::new(640, 480)).unwrap(), &h264_avc, 3, ); @@ -895,7 +885,7 @@ mod tests { #[test] fn rejects_pending_access_unit_over_size_cap() { let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, 640, 480).unwrap(); + AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); assert!(parser.push(&[0, 0, 1, 0x65, 0x88]).unwrap().is_none()); let err = parser.push(&vec![0xff; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); @@ -907,7 +897,7 @@ mod tests { #[test] fn avc_rejects_pending_access_unit_over_size_cap() { - let mut parser = AvcAccessUnitParser::new(4, 0, 33_333, 640, 480).unwrap(); + let mut parser = AvcAccessUnitParser::new(4, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); let nal_len = (MAX_PENDING_ACCESS_UNIT_BYTES + 1) as u32; assert!(parser.push(&nal_len.to_be_bytes()).unwrap().is_none()); diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index 41787d8b1..e127839fd 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -22,7 +22,7 @@ use livekit::{ }, }; -use crate::error::CaptureError; +use crate::{error::CaptureError, primitive::VideoResolution}; const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; @@ -214,10 +214,8 @@ pub struct EncodedAccessUnit<'a> { pub timestamp_us: i64, /// Encoded frame type. pub frame_type: EncodedFrameType, - /// Encoded frame width in pixels. - pub width: u32, - /// Encoded frame height in pixels. - pub height: u32, + /// Encoded frame resolution in pixels. + pub resolution: VideoResolution, /// Optional layer identifiers. pub layers: EncodedLayerInfo, /// Optional codec-specific metadata. @@ -235,10 +233,8 @@ pub struct OwnedEncodedAccessUnit { pub timestamp_us: i64, /// Encoded frame type. pub frame_type: EncodedFrameType, - /// Encoded frame width in pixels. - pub width: u32, - /// Encoded frame height in pixels. - pub height: u32, + /// Encoded frame resolution in pixels. + pub resolution: VideoResolution, /// Optional layer identifiers. pub layers: EncodedLayerInfo, /// Optional codec-specific metadata. @@ -252,16 +248,14 @@ impl OwnedEncodedAccessUnit { payload: impl Into, timestamp_us: i64, frame_type: EncodedFrameType, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Self { Self { codec, payload: payload.into(), timestamp_us, frame_type, - width, - height, + resolution, layers: EncodedLayerInfo::default(), codec_specific: CodecSpecific::None, } @@ -274,8 +268,7 @@ impl OwnedEncodedAccessUnit { payload: EncodedPayload::Contiguous(&self.payload), timestamp_us: self.timestamp_us, frame_type: self.frame_type, - width: self.width, - height: self.height, + resolution: self.resolution, layers: self.layers, codec_specific: self.codec_specific.clone(), } @@ -288,8 +281,7 @@ impl OwnedEncodedAccessUnit { payload: Bytes::from(access_unit.payload.to_vec()), timestamp_us: access_unit.timestamp_us, frame_type: access_unit.frame_type, - width: access_unit.width, - height: access_unit.height, + resolution: access_unit.resolution, layers: access_unit.layers, codec_specific: access_unit.codec_specific.clone(), } @@ -303,16 +295,14 @@ impl<'a> EncodedAccessUnit<'a> { payload: &'a [u8], timestamp_us: i64, frame_type: EncodedFrameType, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Self { Self { codec, payload: EncodedPayload::Contiguous(payload), timestamp_us, frame_type, - width, - height, + resolution, layers: EncodedLayerInfo::default(), codec_specific: CodecSpecific::None, } @@ -322,28 +312,25 @@ impl<'a> EncodedAccessUnit<'a> { pub fn from_h264_nalus( nal_units: &[&[u8]], timestamp_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result, CaptureError> { - Self::from_nalus(EncodedVideoCodec::H264, nal_units, timestamp_us, width, height) + Self::from_nalus(EncodedVideoCodec::H264, nal_units, timestamp_us, resolution) } /// Creates an H.265 access unit from raw NAL-unit payloads. pub fn from_h265_nalus( nal_units: &[&[u8]], timestamp_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result, CaptureError> { - Self::from_nalus(EncodedVideoCodec::H265, nal_units, timestamp_us, width, height) + Self::from_nalus(EncodedVideoCodec::H265, nal_units, timestamp_us, resolution) } fn from_nalus( codec: EncodedVideoCodec, nal_units: &[&[u8]], timestamp_us: i64, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result, CaptureError> { let is_key = is_keyframe_nalus(codec, nal_units)?; Ok(EncodedAccessUnit { @@ -351,8 +338,7 @@ impl<'a> EncodedAccessUnit<'a> { payload: EncodedPayload::Owned(annex_b_payload(nal_units)?), timestamp_us, frame_type: if is_key { EncodedFrameType::Key } else { EncodedFrameType::Delta }, - width, - height, + resolution, layers: EncodedLayerInfo::default(), codec_specific: CodecSpecific::default_for(codec), }) @@ -468,7 +454,7 @@ mod tests { fn h264_nal_helper_assembles_annex_b_and_detects_keyframe() { let sps = [0x67, 1, 2, 3]; let idr = [0x65, 4, 5, 6]; - let au = EncodedAccessUnit::from_h264_nalus(&[&sps, &idr], 10, 640, 480).unwrap(); + let au = EncodedAccessUnit::from_h264_nalus(&[&sps, &idr], 10, VideoResolution::new(640, 480)).unwrap(); assert_eq!(au.codec, EncodedVideoCodec::H264); assert_eq!(au.frame_type, EncodedFrameType::Key); @@ -485,13 +471,13 @@ mod tests { let pps = [0x44, 1, 2]; let idr_w_radl = [19 << 1, 1, 3]; let idr_without_headers = - EncodedAccessUnit::from_h265_nalus(&[&vps, &idr_w_radl], 10, 640, 480).unwrap(); + EncodedAccessUnit::from_h265_nalus(&[&vps, &idr_w_radl], 10, VideoResolution::new(640, 480)).unwrap(); let key = - EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &idr_w_radl], 10, 640, 480) + EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &idr_w_radl], 10, VideoResolution::new(640, 480)) .unwrap(); let cra = [21 << 1, 1, 3]; let cra_with_headers = - EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &cra], 10, 640, 480).unwrap(); + EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &cra], 10, VideoResolution::new(640, 480)).unwrap(); assert_eq!(idr_without_headers.codec, EncodedVideoCodec::H265); assert_eq!(idr_without_headers.frame_type, EncodedFrameType::Delta); @@ -501,7 +487,7 @@ mod tests { #[test] fn h265_rejects_too_short_nal_header() { - let err = EncodedAccessUnit::from_h265_nalus(&[&[0x26]], 10, 640, 480).unwrap_err(); + let err = EncodedAccessUnit::from_h265_nalus(&[&[0x26]], 10, VideoResolution::new(640, 480)).unwrap_err(); assert_eq!(err, CaptureError::H265NalTooShort); } @@ -519,8 +505,7 @@ mod tests { Bytes::from_static(&[1, 2, 3]), 10, EncodedFrameType::Delta, - 640, - 480, + VideoResolution::new(640, 480), ); let borrowed = owned.as_access_unit(); diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 6f64eec61..8300cb7b0 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -16,16 +16,18 @@ pub mod encoded; mod error; +pub mod primitive; pub mod pump; pub mod source; pub mod sources; +pub use primitive::VideoResolution; pub use pump::{ EncodedVideoPump, PixelVideoPump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump, }; pub use source::{ EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, - SourceError, VideoResolution, + SourceError, }; #[cfg(feature = "demo")] pub use sources::demo::{DemoSource, DemoSourceConfig}; diff --git a/livekit-capture/src/primitive.rs b/livekit-capture/src/primitive.rs new file mode 100644 index 000000000..0a0e1b05f --- /dev/null +++ b/livekit-capture/src/primitive.rs @@ -0,0 +1,69 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Domain-neutral video primitives shared across capture paths and backends. +//! +//! These types carry no capture- or codec-specific semantics, so they can serve +//! as a common vocabulary for frame geometry and related quantities across +//! crates. + +// TODO: in a future refactor, move these types into their own +// crate (e.g., `livekit-video-primitives`) so all crates in this workspace can work +// with common types without creating undesirable dependencies. + +/// Pixel dimensions of a video frame. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub struct VideoResolution { + /// Frame width in pixels. + pub width: u32, + /// Frame height in pixels. + pub height: u32, +} + +impl VideoResolution { + /// Creates a video resolution from a width and height in pixels. + /// + /// ``` + /// # use livekit_capture::primitive::VideoResolution; + /// let resolution = VideoResolution::new(1920, 1080); + /// assert_eq!(resolution.width, 1920); + /// assert_eq!(resolution.height, 1080); + /// ``` + pub const fn new(width: u32, height: u32) -> Self { + Self { width, height } + } + + /// Returns the ratio between the width and height components. + /// + /// If the height component is zero, the result is `None`. + /// + /// ``` + /// # use livekit_capture::primitive::VideoResolution; + /// assert_eq!(VideoResolution::new(1920, 960).aspect_ratio(), Some(2.0)); + /// assert_eq!(VideoResolution::new(1920, 0).aspect_ratio(), None); + /// ``` + /// + pub fn aspect_ratio(&self) -> Option { + if self.height == 0 { + return None; + } + Some(f64::from(self.width) / f64::from(self.height)) + } +} + +impl From for livekit::webrtc::video_source::VideoResolution { + fn from(value: VideoResolution) -> Self { + Self { width: value.width, height: value.height } + } +} diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index bd779b714..fc0cadeaa 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -42,10 +42,7 @@ use livekit::{ options::{TrackPublishOptions, VideoEncoderBackend}, webrtc::{ video_frame::{EncodedVideoFrame, I420Buffer, VideoFrame, VideoRotation}, - video_source::{ - native::NativeVideoSource, EncodedRateControl, RtcVideoSource, - VideoResolution as RtcVideoResolution, - }, + video_source::{native::NativeVideoSource, EncodedRateControl, RtcVideoSource}, }, }; use thiserror::Error; @@ -53,9 +50,10 @@ use thiserror::Error; use crate::{ encoded::{CodecSpecific, EncodedFrameType, EncodedLayerInfo, OwnedEncodedAccessUnit}, error::CaptureError, + primitive::VideoResolution, source::{ EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, - SourceError, VideoResolution, + SourceError, }, }; @@ -65,11 +63,6 @@ impl From for RateControl { } } -impl From for RtcVideoResolution { - fn from(resolution: VideoResolution) -> Self { - Self { width: resolution.width, height: resolution.height } - } -} /// Error returned by a pump run. #[derive(Debug, Error)] @@ -431,13 +424,14 @@ fn capture_pixel_frame( fn i420_buffer(frame: &PixelVideoFrame) -> Result { let PixelVideoData::I420 { y, u, v, stride_y, stride_u, stride_v } = &frame.data; - let mut buffer = I420Buffer::new(frame.width, frame.height); - let chroma_width = frame.width.div_ceil(2); - let chroma_height = frame.height.div_ceil(2); + let VideoResolution { width, height } = frame.resolution; + let mut buffer = I420Buffer::new(width, height); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); let (dst_stride_y, dst_stride_u, dst_stride_v) = buffer.strides(); let (dst_y, dst_u, dst_v) = buffer.data_mut(); - copy_plane(y, *stride_y, dst_y, dst_stride_y, frame.width, frame.height)?; + copy_plane(y, *stride_y, dst_y, dst_stride_y, width, height)?; copy_plane(u, *stride_u, dst_u, dst_stride_u, chroma_width, chroma_height)?; copy_plane(v, *stride_v, dst_v, dst_stride_v, chroma_width, chroma_height)?; Ok(buffer) @@ -480,7 +474,7 @@ fn capture_access_unit( payload: &access_unit.payload, timestamp_us: access_unit.timestamp_us, frame_type: access_unit.frame_type.into(), - resolution: RtcVideoResolution { width: access_unit.width, height: access_unit.height }, + resolution: access_unit.resolution.into(), frame_metadata: None, }; rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) @@ -532,8 +526,7 @@ mod tests { let chroma_width = RESOLUTION.width.div_ceil(2); let chroma_height = RESOLUTION.height.div_ceil(2); PixelVideoFrame { - width: RESOLUTION.width, - height: RESOLUTION.height, + resolution: RESOLUTION, timestamp_us, data: PixelVideoData::I420 { y: Bytes::from(vec![128; (RESOLUTION.width * RESOLUTION.height) as usize]), @@ -596,8 +589,7 @@ mod tests { vec![1, 2, 3], timestamp_us, frame_type, - RESOLUTION.width, - RESOLUTION.height, + RESOLUTION, ) } diff --git a/livekit-capture/src/source.rs b/livekit-capture/src/source.rs index 6f0ce0078..342306010 100644 --- a/livekit-capture/src/source.rs +++ b/livekit-capture/src/source.rs @@ -28,16 +28,10 @@ use std::{error::Error, fmt}; use bytes::Bytes; -use crate::encoded::{EncodedVideoCodec, OwnedEncodedAccessUnit}; - -/// Video resolution in pixels. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct VideoResolution { - /// Frame width in pixels. - pub width: u32, - /// Frame height in pixels. - pub height: u32, -} +use crate::{ + encoded::{EncodedVideoCodec, OwnedEncodedAccessUnit}, + primitive::VideoResolution, +}; /// Encoder rate-control target forwarded from WebRTC to an encoded source. #[derive(Debug, Clone, Copy, PartialEq)] @@ -99,10 +93,8 @@ pub enum PixelVideoData { /// One pixel video frame produced by a [`PixelVideoSource`]. #[derive(Debug, Clone)] pub struct PixelVideoFrame { - /// Frame width in pixels. - pub width: u32, - /// Frame height in pixels. - pub height: u32, + /// Frame resolution in pixels. + pub resolution: VideoResolution, /// Capture timestamp in microseconds. pub timestamp_us: i64, /// Pixel data. diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index fd7cc9e1f..c2a3b630d 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -21,8 +21,9 @@ use std::{ use bytes::Bytes; -use crate::source::{ - PixelVideoData, PixelVideoFrame, PixelVideoSource, SourceError, VideoResolution, +use crate::{ + primitive::VideoResolution, + source::{PixelVideoData, PixelVideoFrame, PixelVideoSource, SourceError}, }; /// Colors the demo source cycles through, as `(r, g, b)`. @@ -138,10 +139,9 @@ impl PixelVideoSource for DemoSource { let (y, u, v) = self.planes[color_index % self.planes.len()].clone(); self.frame_index += 1; - let VideoResolution { width, height } = self.config.resolution; + let width = self.config.resolution.width; Ok(Some(PixelVideoFrame { - width, - height, + resolution: self.config.resolution, timestamp_us, data: PixelVideoData::I420 { y, @@ -181,7 +181,7 @@ mod tests { let mut source = DemoSource::new(test_config()); let frame = source.next_frame().unwrap().unwrap(); - assert_eq!((frame.width, frame.height), (64, 36)); + assert_eq!(frame.resolution, VideoResolution::new(64, 36)); let PixelVideoData::I420 { y, u, v, .. } = &frame.data; assert_eq!(y.len(), 64 * 36); diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 4664cca7a..ceb3c26ae 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -26,7 +26,8 @@ use crate::{ CodecSpecific, EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, }, error::CaptureError, - source::{EncodedVideoSource, RateControl, SourceError, VideoResolution}, + primitive::VideoResolution, + source::{EncodedVideoSource, RateControl, SourceError}, }; /// Encoded sample format expected from a GStreamer appsink. @@ -209,8 +210,7 @@ impl GStreamerVideoSource { payload, timestamp_us, frame_type, - self.config.resolution.width, - self.config.resolution.height, + self.config.resolution, ) .map_err(GStreamerVideoSourceError::Capture) } @@ -336,26 +336,23 @@ fn access_unit_from_sample_payload( payload: &[u8], timestamp_us: i64, frame_type: EncodedFrameType, - width: u32, - height: u32, + resolution: VideoResolution, ) -> Result { match sample_format { GStreamerSampleFormat::H264AnnexB => access_unit_from_annex_b( EncodedVideoCodec::H264, Bytes::copy_from_slice(payload), timestamp_us, - width, - height, + resolution, ), GStreamerSampleFormat::H264Avc { nal_length_size } => { - access_unit_from_h264_avc(payload, nal_length_size, timestamp_us, width, height) + access_unit_from_h264_avc(payload, nal_length_size, timestamp_us, resolution) } GStreamerSampleFormat::H265AnnexB => access_unit_from_annex_b( EncodedVideoCodec::H265, Bytes::copy_from_slice(payload), timestamp_us, - width, - height, + resolution, ), GStreamerSampleFormat::AccessUnit { codec } => { if payload.is_empty() { @@ -367,8 +364,7 @@ fn access_unit_from_sample_payload( Bytes::copy_from_slice(payload), timestamp_us, frame_type, - width, - height, + resolution, ); access_unit.codec_specific = CodecSpecific::default_for(codec); Ok(access_unit) @@ -711,8 +707,7 @@ mod tests { &[0, 0, 1, 0x65, 1, 2], 1_000, EncodedFrameType::Delta, - 640, - 480, + VideoResolution::new(640, 480), ) .unwrap(); @@ -728,8 +723,7 @@ mod tests { &[0, 0, 0, 3, 0x65, 1, 2], 1_000, EncodedFrameType::Delta, - 640, - 480, + VideoResolution::new(640, 480), ) .unwrap(); @@ -745,8 +739,7 @@ mod tests { &[1, 2, 3], 2_000, EncodedFrameType::Delta, - 640, - 480, + VideoResolution::new(640, 480), ) .unwrap(); @@ -765,8 +758,7 @@ mod tests { &[1, 2, 3], 2_000, EncodedFrameType::Key, - 640, - 480, + VideoResolution::new(640, 480), ) .unwrap(); assert_eq!(vp9.codec_specific, CodecSpecific::default_for(EncodedVideoCodec::VP9)); @@ -776,8 +768,7 @@ mod tests { &[1, 2, 3], 2_000, EncodedFrameType::Key, - 640, - 480, + VideoResolution::new(640, 480), ) .unwrap(); assert_eq!(av1.codec_specific, CodecSpecific::default_for(EncodedVideoCodec::AV1)); From ef7b862f6bcafe67aa95a84e32b510f91cc0b3b5 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:57:51 -0700 Subject: [PATCH 20/56] Use new module organization --- livekit-capture/README.md | 22 +- livekit-capture/src/encoded/mod.rs | 80 +++- livekit-capture/src/encoded/pump.rs | 274 +++++++++++ livekit-capture/src/error.rs | 31 ++ livekit-capture/src/lib.rs | 22 +- livekit-capture/src/pixel/mod.rs | 86 ++++ livekit-capture/src/pixel/pump.rs | 338 +++++++++++++ livekit-capture/src/pump.rs | 573 +---------------------- livekit-capture/src/source.rs | 176 ------- livekit-capture/src/sources/demo.rs | 3 +- livekit-capture/src/sources/gstreamer.rs | 6 +- 11 files changed, 839 insertions(+), 772 deletions(-) create mode 100644 livekit-capture/src/encoded/pump.rs create mode 100644 livekit-capture/src/pixel/mod.rs create mode 100644 livekit-capture/src/pixel/pump.rs delete mode 100644 livekit-capture/src/source.rs diff --git a/livekit-capture/README.md b/livekit-capture/README.md index b3128fc59..b07712ac3 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -6,20 +6,22 @@ ingest source; the `demo` feature adds a synthetic pixel source for testing. ## Library entry points -- `source::PixelVideoSource` and `source::EncodedVideoSource` — the +- `pixel::PixelVideoSource` and `encoded::EncodedVideoSource` — the libwebrtc-free traits a capture backend implements: pixel sources produce frames published through the WebRTC encoder, encoded sources produce access units published as passthrough. Both traits are object-safe and implemented for `Box`, so sources can be constructed dynamically - and driven through the same pumps. -- `pump::PixelVideoPump` and `pump::EncodedVideoPump` — bridge a source into a - publishable RTC track: each builds the matching `NativeVideoSource`, - derives publish options (`EncodedVideoPump` selects the passthrough encoder), - and runs the capture loop on a plain thread. Encoded pumps forward - downstream keyframe and rate-control requests back to the source and drop - pre-roll deltas until the first keyframe. Both spawn into the same - `pump::RunningPump`, so an application supervises running pumps of either - kind uniformly (`stop()`, `join_async()`, stats). + and driven through the same pumps. Each kind module also holds that kind's + vocabulary (`pixel::PixelVideoFrame`, `encoded::EncodedAccessUnit`, …). +- `pixel::PixelVideoPump` and `encoded::EncodedVideoPump` — bridge a + source into a publishable RTC track: each builds the matching + `NativeVideoSource`, derives publish options (`EncodedVideoPump` selects + the passthrough encoder), and runs the capture loop on a plain thread. + Encoded pumps forward downstream keyframe and rate-control requests back + to the source and drop pre-roll deltas until the first keyframe. Both + spawn into the same `pump::RunningPump`, so an application supervises + running pumps of either kind uniformly (`stop()`, `join_async()`, stats); + the `pump` module holds this shared machinery. - `sources::gstreamer::ensure_encoded_appsink` and friends turn an arbitrary pipeline (containing `appsink name=lk_appsink` or one unlinked encoded pad) into an encoded source; `encoded_caps_string` is the single per-codec caps diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index e127839fd..bbe9e423a 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -12,9 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Encoded video: codec vocabulary, access units, the source contract for +//! pre-encoded ingest, and the pump. +//! +//! Sources produce crate-owned access units independent of libwebrtc and +//! receive crate-owned feedback types, and [`EncodedVideoPump`] bridges them +//! into an RTC track as passthrough. The source trait is object-safe and +//! implemented for `Box`, so sources can be constructed dynamically +//! and driven through the same generic pump. + pub mod h26x; +mod pump; use bytes::Bytes; + +pub use pump::EncodedVideoPump; use livekit::{ options::VideoCodec, webrtc::video_frame::{ @@ -22,7 +34,10 @@ use livekit::{ }, }; -use crate::{error::CaptureError, primitive::VideoResolution}; +use crate::{ + error::{CaptureError, SourceError}, + primitive::VideoResolution, +}; const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; @@ -411,6 +426,69 @@ impl From for RtcEncodedFrameType { } } +/// Encoder rate-control target forwarded from WebRTC to an encoded source. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct RateControl { + /// Target bitrate in bits per second. + pub target_bitrate_bps: u64, + /// Target frame rate in frames per second. + pub framerate_fps: f64, +} + +/// Source of pre-encoded video access units, such as an encoding pipeline. +pub trait EncodedVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Codec produced by this source; fixed for the source's lifetime. + fn codec(&self) -> EncodedVideoCodec; + + /// Blocks until the next access unit is available, returning `Ok(None)` + /// when the source reaches the end of its stream. + fn next_access_unit(&mut self) -> Result, SourceError>; + + /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to + /// the producer so it can emit an IDR. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn request_keyframe(&mut self) {} + + /// Forwards a downstream rate-control target to the producer. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn update_rate_control(&mut self, _target: RateControl) {} +} + +impl EncodedVideoSource for Box { + fn resolution(&self) -> VideoResolution { + (**self).resolution() + } + + fn codec(&self) -> EncodedVideoCodec { + (**self).codec() + } + + fn next_access_unit(&mut self) -> Result, SourceError> { + (**self).next_access_unit() + } + + fn request_keyframe(&mut self) { + (**self).request_keyframe() + } + + fn update_rate_control(&mut self, target: RateControl) { + (**self).update_rate_control(target) + } +} + +// Object safety is part of this trait's contract: dynamic applications box +// sources at their edge and drive them through the same generic pumps. +const _: () = { + fn _assert_object_safe(_: &dyn EncodedVideoSource) {} +}; + pub(crate) fn h264_nal_type(nal: &[u8]) -> Result { let header = nal.first().ok_or(CaptureError::EmptyPayload)?; Ok(header & 0x1f) diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs new file mode 100644 index 000000000..4f360d29c --- /dev/null +++ b/livekit-capture/src/encoded/pump.rs @@ -0,0 +1,274 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pumps encoded access units from a capture source into an RTC video +//! source. + +use std::{fmt, io}; + +use livekit::{ + options::{TrackPublishOptions, VideoEncoderBackend}, + webrtc::{ + video_frame::EncodedVideoFrame, + video_source::{native::NativeVideoSource, EncodedRateControl, RtcVideoSource}, + }, +}; + +use crate::{ + encoded::{ + CodecSpecific, EncodedFrameType, EncodedLayerInfo, EncodedVideoSource, + OwnedEncodedAccessUnit, RateControl, + }, + error::CaptureError, + pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, +}; + +impl From for RateControl { + fn from(target: EncodedRateControl) -> Self { + Self { target_bitrate_bps: target.target_bitrate_bps, framerate_fps: target.framerate_fps } + } +} + +/// Pumps an [`EncodedVideoSource`] into an RTC video source, publishing +/// access units as passthrough. +/// +/// Downstream keyframe requests (PLI/FIR, late subscriber) and rate-control +/// targets are polled between access units and forwarded to the source. +/// Pre-roll delta frames are dropped until the first keyframe, since +/// decoding can only start at a keyframe. +pub struct EncodedVideoPump { + source: S, + rtc_source: NativeVideoSource, + stop: PumpStop, +} + +impl EncodedVideoPump { + /// Creates a pump for an encoded source, building the matching RTC + /// source. + pub fn new(source: S) -> Self { + let rtc_source = NativeVideoSource::new_encoded(source.resolution().into()); + Self { source, rtc_source, stop: PumpStop::new() } + } + + /// Returns the RTC source to create the local track with. + pub fn rtc_source(&self) -> RtcVideoSource { + RtcVideoSource::Native(self.rtc_source.clone()) + } + + /// Returns publish options for encoded passthrough. + pub fn publish_options(&self) -> TrackPublishOptions { + TrackPublishOptions { + video_codec: self.source.codec().into(), + video_encoder: VideoEncoderBackend::PreEncoded, + simulcast: false, + ..Default::default() + } + } + + /// Returns a cancellation handle for this pump. + pub fn stop_handle(&self) -> PumpStop { + self.stop.clone() + } + + /// Returns the underlying capture source. + pub fn source(&self) -> &S { + &self.source + } + + /// Returns the underlying capture source mutably. + pub fn source_mut(&mut self) -> &mut S { + &mut self.source + } + + /// Runs the pump on the calling thread until the source ends, a failure, + /// or the stop handle fires. + /// + /// Sources block, so callers on an async runtime should run this on a + /// dedicated thread (see [`EncodedVideoPump::spawn`]) or a blocking pool. + pub fn run(mut self) -> Result { + let mut frames_captured = 0; + let mut awaiting_initial_keyframe = true; + let exit = loop { + if self.stop.is_stopped() { + break PumpExit::Stopped; + } + if let Some(target) = self.rtc_source.take_rate_control_request() { + self.source.update_rate_control(target.into()); + } + if self.rtc_source.take_keyframe_request() { + self.source.request_keyframe(); + } + + let Some(access_unit) = self.source.next_access_unit()? else { + break PumpExit::EndOfStream; + }; + + // Drop pre-roll deltas: decoding can only start at a keyframe. + if awaiting_initial_keyframe && access_unit.frame_type != EncodedFrameType::Key { + continue; + } + awaiting_initial_keyframe = false; + + capture_access_unit(&self.rtc_source, &access_unit)?; + frames_captured += 1; + }; + Ok(PumpStats { frames_captured, exit }) + } + + /// Runs the pump on a dedicated thread. + /// + /// Panics on the pump thread are caught and reported as + /// [`PumpError::Panicked`] when the pump is joined. + pub fn spawn(self) -> io::Result + where + S: 'static, + { + let stop = self.stop_handle(); + spawn_pump(stop, move || self.run()) + } +} + +impl fmt::Debug for EncodedVideoPump { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EncodedVideoPump") + .field("rtc_source", &self.rtc_source) + .field("stop", &self.stop) + .finish_non_exhaustive() + } +} + +fn capture_access_unit( + rtc_source: &NativeVideoSource, + access_unit: &OwnedEncodedAccessUnit, +) -> Result<(), CaptureError> { + validate_access_unit(access_unit)?; + + let frame = EncodedVideoFrame { + codec: access_unit.codec.into(), + payload: &access_unit.payload, + timestamp_us: access_unit.timestamp_us, + frame_type: access_unit.frame_type.into(), + resolution: access_unit.resolution.into(), + frame_metadata: None, + }; + rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) +} + +/// The passthrough path forwards single-layer streams: access units carrying +/// temporal/spatial layer ids or layering metadata are rejected so callers +/// are not misled into thinking that metadata reaches the wire. +fn validate_access_unit(access_unit: &OwnedEncodedAccessUnit) -> Result<(), CaptureError> { + if access_unit.payload.is_empty() { + return Err(CaptureError::EmptyPayload); + } + if access_unit.layers != EncodedLayerInfo::default() { + return Err(CaptureError::UnsupportedLayeredEncoding( + "temporal/spatial layer ids are not forwarded by the passthrough encoder", + )); + } + if access_unit.codec_specific != CodecSpecific::None + && access_unit.codec_specific != CodecSpecific::default_for(access_unit.codec) + { + return Err(CaptureError::UnsupportedLayeredEncoding( + "codec-specific layering metadata is not forwarded by the passthrough encoder", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use bytes::Bytes; + + use super::*; + use crate::{encoded::EncodedVideoCodec, error::SourceError, primitive::VideoResolution}; + + const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; + + struct FakeEncodedSource { + access_units: VecDeque, + } + + impl FakeEncodedSource { + fn new(access_units: impl IntoIterator) -> Self { + Self { access_units: access_units.into_iter().collect() } + } + } + + impl EncodedVideoSource for FakeEncodedSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn codec(&self) -> EncodedVideoCodec { + EncodedVideoCodec::VP8 + } + + fn next_access_unit(&mut self) -> Result, SourceError> { + Ok(self.access_units.pop_front()) + } + } + + fn access_unit(timestamp_us: i64, frame_type: EncodedFrameType) -> OwnedEncodedAccessUnit { + OwnedEncodedAccessUnit::new( + EncodedVideoCodec::VP8, + vec![1, 2, 3], + timestamp_us, + frame_type, + RESOLUTION, + ) + } + + #[test] + fn encoded_pump_starts_at_initial_keyframe() { + let source = FakeEncodedSource::new([ + access_unit(1, EncodedFrameType::Delta), + access_unit(2, EncodedFrameType::Delta), + access_unit(3, EncodedFrameType::Key), + access_unit(4, EncodedFrameType::Delta), + ]); + let stats = EncodedVideoPump::new(source).run().unwrap(); + assert_eq!(stats.frames_captured, 2); + } + + #[test] + fn boxed_source_drives_generic_pump() { + // The dynamic-instantiation pattern: box at the edge, same pump. + let source: Box = + Box::new(FakeEncodedSource::new([access_unit(1, EncodedFrameType::Key)])); + let pump = EncodedVideoPump::new(source); + assert_eq!(pump.publish_options().video_encoder, VideoEncoderBackend::PreEncoded); + let stats = pump.run().unwrap(); + assert_eq!(stats.frames_captured, 1); + } + + #[test] + fn encoded_pump_rejects_empty_payloads() { + let mut unit = access_unit(1, EncodedFrameType::Key); + unit.payload = Bytes::new(); + + let result = EncodedVideoPump::new(FakeEncodedSource::new([unit])).run(); + assert!(matches!(result, Err(PumpError::Capture(CaptureError::EmptyPayload)))); + } + + #[test] + fn encoded_publish_options_use_passthrough() { + let pump = EncodedVideoPump::new(FakeEncodedSource::new([])); + let options = pump.publish_options(); + assert_eq!(options.video_encoder, VideoEncoderBackend::PreEncoded); + assert!(!options.simulcast); + } +} diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index 47be05d3f..ee6d2752b 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -12,10 +12,41 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Error types shared across capture paths. + +use std::{error::Error as StdError, fmt}; + use thiserror::Error; use crate::encoded::{EncodedVideoCodec, EncodedWireFormat}; +/// Error returned by a capture source. +/// +/// Backend-specific errors are type-erased so sources stay usable as trait +/// objects; the wrapped error remains reachable for display and through +/// [`StdError::source`]. +#[derive(Debug)] +pub struct SourceError(Box); + +impl SourceError { + /// Wraps a backend error. + pub fn new(error: impl Into>) -> Self { + Self(error.into()) + } +} + +impl fmt::Display for SourceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl StdError for SourceError { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + self.0.source() + } +} + /// Error returned by capture helpers. #[derive(Debug, Error, PartialEq, Eq)] pub enum CaptureError { diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 8300cb7b0..1bae9f094 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -15,26 +15,8 @@ //! Capture sources and helpers for publishing video with LiveKit. pub mod encoded; -mod error; +pub mod error; +pub mod pixel; pub mod primitive; pub mod pump; -pub mod source; pub mod sources; - -pub use primitive::VideoResolution; -pub use pump::{ - EncodedVideoPump, PixelVideoPump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump, -}; -pub use source::{ - EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, - SourceError, -}; -#[cfg(feature = "demo")] -pub use sources::demo::{DemoSource, DemoSourceConfig}; - -pub use encoded::{ - CodecSpecific, EncodedAccessUnit, EncodedFragment, EncodedFrameType, EncodedLayerInfo, - EncodedPayload, EncodedVideoCodec, EncodedWireFormat, H264PacketizationMode, - OwnedEncodedAccessUnit, -}; -pub use error::CaptureError; diff --git a/livekit-capture/src/pixel/mod.rs b/livekit-capture/src/pixel/mod.rs new file mode 100644 index 000000000..d1c313289 --- /dev/null +++ b/livekit-capture/src/pixel/mod.rs @@ -0,0 +1,86 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pixel (unencoded) video: frame types, the source contract, and the pump. +//! +//! Sources produce crate-owned frame types independent of libwebrtc, and +//! [`PixelVideoPump`] bridges them into an RTC track. The source trait is +//! object-safe and implemented for `Box`, so sources can be +//! constructed dynamically and driven through the same generic pump. + +mod pump; + +use bytes::Bytes; + +pub use pump::PixelVideoPump; + +use crate::{error::SourceError, primitive::VideoResolution}; + +/// Pixel data of one video frame. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum PixelVideoData { + /// Planar YUV 4:2:0 with 8-bit samples. + I420 { + /// Luma plane. + y: Bytes, + /// Blue-difference chroma plane. + u: Bytes, + /// Red-difference chroma plane. + v: Bytes, + /// Luma plane stride in bytes. + stride_y: u32, + /// U plane stride in bytes. + stride_u: u32, + /// V plane stride in bytes. + stride_v: u32, + }, +} + +/// One pixel video frame produced by a [`PixelVideoSource`]. +#[derive(Debug, Clone)] +pub struct PixelVideoFrame { + /// Frame resolution in pixels. + pub resolution: VideoResolution, + /// Capture timestamp in microseconds. + pub timestamp_us: i64, + /// Pixel data. + pub data: PixelVideoData, +} + +/// Source of pixel (unencoded) video frames, such as a camera device. +pub trait PixelVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Blocks until the next frame is available, returning `Ok(None)` when + /// the source reaches the end of its stream. + fn next_frame(&mut self) -> Result, SourceError>; +} + +impl PixelVideoSource for Box { + fn resolution(&self) -> VideoResolution { + (**self).resolution() + } + + fn next_frame(&mut self) -> Result, SourceError> { + (**self).next_frame() + } +} + +// Object safety is part of this trait's contract: dynamic applications box +// sources at their edge and drive them through the same generic pumps. +const _: () = { + fn _assert_object_safe(_: &dyn PixelVideoSource) {} +}; diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs new file mode 100644 index 000000000..6e662ebcf --- /dev/null +++ b/livekit-capture/src/pixel/pump.rs @@ -0,0 +1,338 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pumps pixel frames from a capture source into an RTC video source. + +use std::{fmt, io}; + +use livekit::{ + options::TrackPublishOptions, + webrtc::{ + video_frame::{I420Buffer, VideoFrame, VideoRotation}, + video_source::{native::NativeVideoSource, RtcVideoSource}, + }, +}; + +use crate::{ + error::CaptureError, + pixel::{PixelVideoData, PixelVideoFrame, PixelVideoSource}, + primitive::VideoResolution, + pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, +}; + +/// Pumps a [`PixelVideoSource`] into an RTC video source, publishing frames +/// through the WebRTC encoder. +pub struct PixelVideoPump { + source: S, + rtc_source: NativeVideoSource, + stop: PumpStop, +} + +impl PixelVideoPump { + /// Creates a pump for a pixel source, building the matching RTC source. + /// + /// This must be called from the context of the async runtime driving the + /// SDK, because the RTC source spawns its keepalive task at construction. + /// The pump itself runs on plain threads. + pub fn new(source: S) -> Self { + let rtc_source = NativeVideoSource::new(source.resolution().into(), false); + Self { source, rtc_source, stop: PumpStop::new() } + } + + /// Returns the RTC source to create the local track with. + pub fn rtc_source(&self) -> RtcVideoSource { + RtcVideoSource::Native(self.rtc_source.clone()) + } + + /// Returns publish options appropriate for a pixel source. + pub fn publish_options(&self) -> TrackPublishOptions { + TrackPublishOptions::default() + } + + /// Returns a cancellation handle for this pump. + pub fn stop_handle(&self) -> PumpStop { + self.stop.clone() + } + + /// Returns the underlying capture source. + pub fn source(&self) -> &S { + &self.source + } + + /// Returns the underlying capture source mutably. + pub fn source_mut(&mut self) -> &mut S { + &mut self.source + } + + /// Runs the pump on the calling thread until the source ends, a failure, + /// or the stop handle fires. + /// + /// Sources block, so callers on an async runtime should run this on a + /// dedicated thread (see [`PixelVideoPump::spawn`]) or a blocking pool. + pub fn run(mut self) -> Result { + let mut frames_captured = 0; + let exit = loop { + if self.stop.is_stopped() { + break PumpExit::Stopped; + } + let Some(frame) = self.source.next_frame()? else { + break PumpExit::EndOfStream; + }; + capture_pixel_frame(&self.rtc_source, &frame)?; + frames_captured += 1; + }; + Ok(PumpStats { frames_captured, exit }) + } + + /// Runs the pump on a dedicated thread. + /// + /// Panics on the pump thread are caught and reported as + /// [`PumpError::Panicked`] when the pump is joined. + pub fn spawn(self) -> io::Result + where + S: 'static, + { + let stop = self.stop_handle(); + spawn_pump(stop, move || self.run()) + } +} + +impl fmt::Debug for PixelVideoPump { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PixelVideoPump") + .field("rtc_source", &self.rtc_source) + .field("stop", &self.stop) + .finish_non_exhaustive() + } +} + +fn capture_pixel_frame( + rtc_source: &NativeVideoSource, + frame: &PixelVideoFrame, +) -> Result<(), CaptureError> { + let buffer = i420_buffer(frame)?; + rtc_source.capture_frame(&VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: frame.timestamp_us, + frame_metadata: None, + buffer, + }); + Ok(()) +} + +fn i420_buffer(frame: &PixelVideoFrame) -> Result { + let PixelVideoData::I420 { y, u, v, stride_y, stride_u, stride_v } = &frame.data; + + let VideoResolution { width, height } = frame.resolution; + let mut buffer = I420Buffer::new(width, height); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let (dst_stride_y, dst_stride_u, dst_stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + + copy_plane(y, *stride_y, dst_y, dst_stride_y, width, height)?; + copy_plane(u, *stride_u, dst_u, dst_stride_u, chroma_width, chroma_height)?; + copy_plane(v, *stride_v, dst_v, dst_stride_v, chroma_width, chroma_height)?; + Ok(buffer) +} + +fn copy_plane( + src: &[u8], + src_stride: u32, + dst: &mut [u8], + dst_stride: u32, + width: u32, + height: u32, +) -> Result<(), CaptureError> { + let (width, height) = (width as usize, height as usize); + let (src_stride, dst_stride) = (src_stride as usize, dst_stride as usize); + if src_stride < width { + return Err(CaptureError::InvalidPixelFrame("plane stride is smaller than its width")); + } + // The final row may be unpadded. + let min_len = (height - 1).saturating_mul(src_stride) + width; + if src.len() < min_len { + return Err(CaptureError::InvalidPixelFrame("plane data is shorter than its dimensions")); + } + + for row in 0..height { + let src_row = &src[row * src_stride..][..width]; + dst[row * dst_stride..][..width].copy_from_slice(src_row); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use bytes::Bytes; + + use super::*; + use crate::error::SourceError; + + const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; + + /// Pixel RTC sources spawn their keepalive task at construction; give the + /// tests the runtime context an SDK application would have. + fn runtime_context() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("failed to build test runtime") + } + + fn pixel_frame(timestamp_us: i64) -> PixelVideoFrame { + let chroma_width = RESOLUTION.width.div_ceil(2); + let chroma_height = RESOLUTION.height.div_ceil(2); + PixelVideoFrame { + resolution: RESOLUTION, + timestamp_us, + data: PixelVideoData::I420 { + y: Bytes::from(vec![128; (RESOLUTION.width * RESOLUTION.height) as usize]), + u: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), + v: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), + stride_y: RESOLUTION.width, + stride_u: chroma_width, + stride_v: chroma_width, + }, + } + } + + struct FakePixelSource { + frames: VecDeque, + } + + impl FakePixelSource { + fn new(frames: impl IntoIterator) -> Self { + Self { frames: frames.into_iter().collect() } + } + } + + impl PixelVideoSource for FakePixelSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + Ok(self.frames.pop_front()) + } + } + + #[test] + fn pixel_pump_captures_all_frames_until_eof() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); + let stats = PixelVideoPump::new(source).run().unwrap(); + assert_eq!(stats.frames_captured, 3); + assert_eq!(stats.exit, PumpExit::EndOfStream); + } + + #[test] + fn boxed_source_drives_generic_pump() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + // The dynamic-instantiation pattern: box at the edge, same pump. + let source: Box = + Box::new(FakePixelSource::new([pixel_frame(1), pixel_frame(2)])); + let stats = PixelVideoPump::new(source).run().unwrap(); + assert_eq!(stats.frames_captured, 2); + } + + #[test] + fn pump_panics_become_errors() { + struct PanickingSource; + + impl PixelVideoSource for PanickingSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + panic!("source exploded"); + } + } + + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let running = PixelVideoPump::new(PanickingSource).spawn().unwrap(); + let error = running.join().unwrap_err(); + assert!( + matches!(&error, PumpError::Panicked(message) if message.contains("source exploded")) + ); + } + + #[test] + fn running_pump_stops_on_signal() { + struct EndlessSource; + + impl PixelVideoSource for EndlessSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok(Some(pixel_frame(0))) + } + } + + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + let stats = running.stop_and_join().unwrap(); + assert!(stats.frames_captured > 0); + assert_eq!(stats.exit, PumpExit::Stopped); + } + + #[test] + fn pixel_pump_rejects_short_planes() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let mut frame = pixel_frame(1); + let PixelVideoData::I420 { y, .. } = &mut frame.data; + *y = Bytes::from(vec![128; 8]); + + let result = PixelVideoPump::new(FakePixelSource::new([frame])).run(); + assert!(matches!(result, Err(PumpError::Capture(CaptureError::InvalidPixelFrame(_))))); + } + + #[tokio::test] + async fn pump_stops_and_joins_async() { + struct EndlessSource; + + impl PixelVideoSource for EndlessSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self) -> Result, SourceError> { + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok(Some(pixel_frame(0))) + } + } + + let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let stats = running.stop_and_join_async().await.unwrap(); + assert!(stats.frames_captured > 0); + } +} diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index fc0cadeaa..31ac468f5 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -12,24 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Pumps frames from a capture source into an RTC video source. +//! Machinery shared by the capture pumps. //! -//! [`PixelVideoPump`] and [`EncodedVideoPump`] are the bridges between the -//! libwebrtc-free source traits in [`source`](crate::source) and a -//! publishable RTC track: each builds the matching [`NativeVideoSource`], -//! converts crate-owned frame types at the boundary, and — for encoded -//! sources — forwards downstream keyframe and rate-control requests back to -//! the producer. -//! -//! Both pumps are generic over a concrete source, so statically-known -//! sources pay for no type erasure. Applications that construct sources -//! dynamically box them at their edge (`PixelVideoPump>`); both pumps spawn into the same [`RunningPump`], so -//! running pumps of either kind are handled uniformly. +//! The kind-specific pumps live with their kinds — +//! [`PixelVideoPump`](crate::pixel::PixelVideoPump) and +//! [`EncodedVideoPump`](crate::encoded::EncodedVideoPump) — and are generic +//! over a concrete source, so statically-known sources pay for no type +//! erasure. Applications that construct sources dynamically box them at +//! their edge (`PixelVideoPump>`). Both pumps +//! spawn into the same [`RunningPump`] defined here, so running pumps of +//! either kind are supervised uniformly. use std::{ any::Any, - fmt, io, + io, panic::{catch_unwind, AssertUnwindSafe}, sync::{ atomic::{AtomicBool, Ordering}, @@ -38,31 +34,9 @@ use std::{ thread, }; -use livekit::{ - options::{TrackPublishOptions, VideoEncoderBackend}, - webrtc::{ - video_frame::{EncodedVideoFrame, I420Buffer, VideoFrame, VideoRotation}, - video_source::{native::NativeVideoSource, EncodedRateControl, RtcVideoSource}, - }, -}; use thiserror::Error; -use crate::{ - encoded::{CodecSpecific, EncodedFrameType, EncodedLayerInfo, OwnedEncodedAccessUnit}, - error::CaptureError, - primitive::VideoResolution, - source::{ - EncodedVideoSource, PixelVideoData, PixelVideoFrame, PixelVideoSource, RateControl, - SourceError, - }, -}; - -impl From for RateControl { - fn from(target: EncodedRateControl) -> Self { - Self { target_bitrate_bps: target.target_bitrate_bps, framerate_fps: target.framerate_fps } - } -} - +use crate::error::{CaptureError, SourceError}; /// Error returned by a pump run. #[derive(Debug, Error)] @@ -121,203 +95,9 @@ impl PumpStop { } } -/// Pumps a [`PixelVideoSource`] into an RTC video source, publishing frames -/// through the WebRTC encoder. -pub struct PixelVideoPump { - source: S, - rtc_source: NativeVideoSource, - stop: PumpStop, -} - -impl PixelVideoPump { - /// Creates a pump for a pixel source, building the matching RTC source. - /// - /// This must be called from the context of the async runtime driving the - /// SDK, because the RTC source spawns its keepalive task at construction. - /// The pump itself runs on plain threads. - pub fn new(source: S) -> Self { - let rtc_source = NativeVideoSource::new(source.resolution().into(), false); - Self { source, rtc_source, stop: PumpStop::new() } - } - - /// Returns the RTC source to create the local track with. - pub fn rtc_source(&self) -> RtcVideoSource { - RtcVideoSource::Native(self.rtc_source.clone()) - } - - /// Returns publish options appropriate for a pixel source. - pub fn publish_options(&self) -> TrackPublishOptions { - TrackPublishOptions::default() - } - - /// Returns a cancellation handle for this pump. - pub fn stop_handle(&self) -> PumpStop { - self.stop.clone() - } - - /// Returns the underlying capture source. - pub fn source(&self) -> &S { - &self.source - } - - /// Returns the underlying capture source mutably. - pub fn source_mut(&mut self) -> &mut S { - &mut self.source - } - - /// Runs the pump on the calling thread until the source ends, a failure, - /// or the stop handle fires. - /// - /// Sources block, so callers on an async runtime should run this on a - /// dedicated thread (see [`PixelVideoPump::spawn`]) or a blocking pool. - pub fn run(mut self) -> Result { - let mut frames_captured = 0; - let exit = loop { - if self.stop.is_stopped() { - break PumpExit::Stopped; - } - let Some(frame) = self.source.next_frame()? else { - break PumpExit::EndOfStream; - }; - capture_pixel_frame(&self.rtc_source, &frame)?; - frames_captured += 1; - }; - Ok(PumpStats { frames_captured, exit }) - } - - /// Runs the pump on a dedicated thread. - /// - /// Panics on the pump thread are caught and reported as - /// [`PumpError::Panicked`] when the pump is joined. - pub fn spawn(self) -> io::Result - where - S: 'static, - { - let stop = self.stop_handle(); - spawn_pump(stop, move || self.run()) - } -} - -impl fmt::Debug for PixelVideoPump { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PixelVideoPump") - .field("rtc_source", &self.rtc_source) - .field("stop", &self.stop) - .finish_non_exhaustive() - } -} - -/// Pumps an [`EncodedVideoSource`] into an RTC video source, publishing -/// access units as passthrough. -/// -/// Downstream keyframe requests (PLI/FIR, late subscriber) and rate-control -/// targets are polled between access units and forwarded to the source. -/// Pre-roll delta frames are dropped until the first keyframe, since -/// decoding can only start at a keyframe. -pub struct EncodedVideoPump { - source: S, - rtc_source: NativeVideoSource, - stop: PumpStop, -} - -impl EncodedVideoPump { - /// Creates a pump for an encoded source, building the matching RTC - /// source. - pub fn new(source: S) -> Self { - let rtc_source = NativeVideoSource::new_encoded(source.resolution().into()); - Self { source, rtc_source, stop: PumpStop::new() } - } - - /// Returns the RTC source to create the local track with. - pub fn rtc_source(&self) -> RtcVideoSource { - RtcVideoSource::Native(self.rtc_source.clone()) - } - - /// Returns publish options for encoded passthrough. - pub fn publish_options(&self) -> TrackPublishOptions { - TrackPublishOptions { - video_codec: self.source.codec().into(), - video_encoder: VideoEncoderBackend::PreEncoded, - simulcast: false, - ..Default::default() - } - } - - /// Returns a cancellation handle for this pump. - pub fn stop_handle(&self) -> PumpStop { - self.stop.clone() - } - - /// Returns the underlying capture source. - pub fn source(&self) -> &S { - &self.source - } - - /// Returns the underlying capture source mutably. - pub fn source_mut(&mut self) -> &mut S { - &mut self.source - } - - /// Runs the pump on the calling thread until the source ends, a failure, - /// or the stop handle fires. - /// - /// Sources block, so callers on an async runtime should run this on a - /// dedicated thread (see [`EncodedVideoPump::spawn`]) or a blocking pool. - pub fn run(mut self) -> Result { - let mut frames_captured = 0; - let mut awaiting_initial_keyframe = true; - let exit = loop { - if self.stop.is_stopped() { - break PumpExit::Stopped; - } - if let Some(target) = self.rtc_source.take_rate_control_request() { - self.source.update_rate_control(target.into()); - } - if self.rtc_source.take_keyframe_request() { - self.source.request_keyframe(); - } - - let Some(access_unit) = self.source.next_access_unit()? else { - break PumpExit::EndOfStream; - }; - - // Drop pre-roll deltas: decoding can only start at a keyframe. - if awaiting_initial_keyframe && access_unit.frame_type != EncodedFrameType::Key { - continue; - } - awaiting_initial_keyframe = false; - - capture_access_unit(&self.rtc_source, &access_unit)?; - frames_captured += 1; - }; - Ok(PumpStats { frames_captured, exit }) - } - - /// Runs the pump on a dedicated thread. - /// - /// Panics on the pump thread are caught and reported as - /// [`PumpError::Panicked`] when the pump is joined. - pub fn spawn(self) -> io::Result - where - S: 'static, - { - let stop = self.stop_handle(); - spawn_pump(stop, move || self.run()) - } -} - -impl fmt::Debug for EncodedVideoPump { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("EncodedVideoPump") - .field("rtc_source", &self.rtc_source) - .field("stop", &self.stop) - .finish_non_exhaustive() - } -} - /// Spawns a pump run on a dedicated thread, wiring panic capture and the /// completion signal shared by both pump kinds. -fn spawn_pump( +pub(crate) fn spawn_pump( stop: PumpStop, run: impl FnOnce() -> Result + Send + 'static, ) -> io::Result { @@ -406,332 +186,3 @@ impl RunningPump { self.join_async().await } } - -fn capture_pixel_frame( - rtc_source: &NativeVideoSource, - frame: &PixelVideoFrame, -) -> Result<(), CaptureError> { - let buffer = i420_buffer(frame)?; - rtc_source.capture_frame(&VideoFrame { - rotation: VideoRotation::VideoRotation0, - timestamp_us: frame.timestamp_us, - frame_metadata: None, - buffer, - }); - Ok(()) -} - -fn i420_buffer(frame: &PixelVideoFrame) -> Result { - let PixelVideoData::I420 { y, u, v, stride_y, stride_u, stride_v } = &frame.data; - - let VideoResolution { width, height } = frame.resolution; - let mut buffer = I420Buffer::new(width, height); - let chroma_width = width.div_ceil(2); - let chroma_height = height.div_ceil(2); - let (dst_stride_y, dst_stride_u, dst_stride_v) = buffer.strides(); - let (dst_y, dst_u, dst_v) = buffer.data_mut(); - - copy_plane(y, *stride_y, dst_y, dst_stride_y, width, height)?; - copy_plane(u, *stride_u, dst_u, dst_stride_u, chroma_width, chroma_height)?; - copy_plane(v, *stride_v, dst_v, dst_stride_v, chroma_width, chroma_height)?; - Ok(buffer) -} - -fn copy_plane( - src: &[u8], - src_stride: u32, - dst: &mut [u8], - dst_stride: u32, - width: u32, - height: u32, -) -> Result<(), CaptureError> { - let (width, height) = (width as usize, height as usize); - let (src_stride, dst_stride) = (src_stride as usize, dst_stride as usize); - if src_stride < width { - return Err(CaptureError::InvalidPixelFrame("plane stride is smaller than its width")); - } - // The final row may be unpadded. - let min_len = (height - 1).saturating_mul(src_stride) + width; - if src.len() < min_len { - return Err(CaptureError::InvalidPixelFrame("plane data is shorter than its dimensions")); - } - - for row in 0..height { - let src_row = &src[row * src_stride..][..width]; - dst[row * dst_stride..][..width].copy_from_slice(src_row); - } - Ok(()) -} - -fn capture_access_unit( - rtc_source: &NativeVideoSource, - access_unit: &OwnedEncodedAccessUnit, -) -> Result<(), CaptureError> { - validate_access_unit(access_unit)?; - - let frame = EncodedVideoFrame { - codec: access_unit.codec.into(), - payload: &access_unit.payload, - timestamp_us: access_unit.timestamp_us, - frame_type: access_unit.frame_type.into(), - resolution: access_unit.resolution.into(), - frame_metadata: None, - }; - rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) -} - -/// The passthrough path forwards single-layer streams: access units carrying -/// temporal/spatial layer ids or layering metadata are rejected so callers -/// are not misled into thinking that metadata reaches the wire. -fn validate_access_unit(access_unit: &OwnedEncodedAccessUnit) -> Result<(), CaptureError> { - if access_unit.payload.is_empty() { - return Err(CaptureError::EmptyPayload); - } - if access_unit.layers != EncodedLayerInfo::default() { - return Err(CaptureError::UnsupportedLayeredEncoding( - "temporal/spatial layer ids are not forwarded by the passthrough encoder", - )); - } - if access_unit.codec_specific != CodecSpecific::None - && access_unit.codec_specific != CodecSpecific::default_for(access_unit.codec) - { - return Err(CaptureError::UnsupportedLayeredEncoding( - "codec-specific layering metadata is not forwarded by the passthrough encoder", - )); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::VecDeque; - - use bytes::Bytes; - - use super::*; - use crate::encoded::EncodedVideoCodec; - - const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; - - /// Pixel RTC sources spawn their keepalive task at construction; give the - /// tests the runtime context an SDK application would have. - fn runtime_context() -> tokio::runtime::Runtime { - tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .expect("failed to build test runtime") - } - - fn pixel_frame(timestamp_us: i64) -> PixelVideoFrame { - let chroma_width = RESOLUTION.width.div_ceil(2); - let chroma_height = RESOLUTION.height.div_ceil(2); - PixelVideoFrame { - resolution: RESOLUTION, - timestamp_us, - data: PixelVideoData::I420 { - y: Bytes::from(vec![128; (RESOLUTION.width * RESOLUTION.height) as usize]), - u: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), - v: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), - stride_y: RESOLUTION.width, - stride_u: chroma_width, - stride_v: chroma_width, - }, - } - } - - struct FakePixelSource { - frames: VecDeque, - } - - impl FakePixelSource { - fn new(frames: impl IntoIterator) -> Self { - Self { frames: frames.into_iter().collect() } - } - } - - impl PixelVideoSource for FakePixelSource { - fn resolution(&self) -> VideoResolution { - RESOLUTION - } - - fn next_frame(&mut self) -> Result, SourceError> { - Ok(self.frames.pop_front()) - } - } - - struct FakeEncodedSource { - access_units: VecDeque, - } - - impl FakeEncodedSource { - fn new(access_units: impl IntoIterator) -> Self { - Self { access_units: access_units.into_iter().collect() } - } - } - - impl EncodedVideoSource for FakeEncodedSource { - fn resolution(&self) -> VideoResolution { - RESOLUTION - } - - fn codec(&self) -> EncodedVideoCodec { - EncodedVideoCodec::VP8 - } - - fn next_access_unit(&mut self) -> Result, SourceError> { - Ok(self.access_units.pop_front()) - } - } - - fn access_unit(timestamp_us: i64, frame_type: EncodedFrameType) -> OwnedEncodedAccessUnit { - OwnedEncodedAccessUnit::new( - EncodedVideoCodec::VP8, - vec![1, 2, 3], - timestamp_us, - frame_type, - RESOLUTION, - ) - } - - #[test] - fn pixel_pump_captures_all_frames_until_eof() { - let runtime = runtime_context(); - let _guard = runtime.enter(); - - let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); - let stats = PixelVideoPump::new(source).run().unwrap(); - assert_eq!(stats.frames_captured, 3); - assert_eq!(stats.exit, PumpExit::EndOfStream); - } - - #[test] - fn boxed_sources_drive_generic_pumps() { - let runtime = runtime_context(); - let _guard = runtime.enter(); - - // The dynamic-instantiation pattern: box at the edge, same pumps. - let source: Box = - Box::new(FakePixelSource::new([pixel_frame(1), pixel_frame(2)])); - let stats = PixelVideoPump::new(source).run().unwrap(); - assert_eq!(stats.frames_captured, 2); - - let source: Box = - Box::new(FakeEncodedSource::new([access_unit(1, EncodedFrameType::Key)])); - let pump = EncodedVideoPump::new(source); - assert_eq!(pump.publish_options().video_encoder, VideoEncoderBackend::PreEncoded); - let stats = pump.run().unwrap(); - assert_eq!(stats.frames_captured, 1); - } - - #[test] - fn pump_panics_become_errors() { - struct PanickingSource; - - impl PixelVideoSource for PanickingSource { - fn resolution(&self) -> VideoResolution { - RESOLUTION - } - - fn next_frame(&mut self) -> Result, SourceError> { - panic!("source exploded"); - } - } - - let runtime = runtime_context(); - let _guard = runtime.enter(); - - let running = PixelVideoPump::new(PanickingSource).spawn().unwrap(); - let error = running.join().unwrap_err(); - assert!( - matches!(&error, PumpError::Panicked(message) if message.contains("source exploded")) - ); - } - - #[test] - fn running_pump_stops_on_signal() { - struct EndlessSource; - - impl PixelVideoSource for EndlessSource { - fn resolution(&self) -> VideoResolution { - RESOLUTION - } - - fn next_frame(&mut self) -> Result, SourceError> { - std::thread::sleep(std::time::Duration::from_millis(1)); - Ok(Some(pixel_frame(0))) - } - } - - let runtime = runtime_context(); - let _guard = runtime.enter(); - - let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); - std::thread::sleep(std::time::Duration::from_millis(20)); - let stats = running.stop_and_join().unwrap(); - assert!(stats.frames_captured > 0); - assert_eq!(stats.exit, PumpExit::Stopped); - } - - #[test] - fn pixel_pump_rejects_short_planes() { - let runtime = runtime_context(); - let _guard = runtime.enter(); - - let mut frame = pixel_frame(1); - let PixelVideoData::I420 { y, .. } = &mut frame.data; - *y = Bytes::from(vec![128; 8]); - - let result = PixelVideoPump::new(FakePixelSource::new([frame])).run(); - assert!(matches!(result, Err(PumpError::Capture(CaptureError::InvalidPixelFrame(_))))); - } - - #[tokio::test] - async fn pump_stops_and_joins_async() { - struct EndlessSource; - - impl PixelVideoSource for EndlessSource { - fn resolution(&self) -> VideoResolution { - RESOLUTION - } - - fn next_frame(&mut self) -> Result, SourceError> { - std::thread::sleep(std::time::Duration::from_millis(1)); - Ok(Some(pixel_frame(0))) - } - } - - let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - let stats = running.stop_and_join_async().await.unwrap(); - assert!(stats.frames_captured > 0); - } - - #[test] - fn encoded_pump_starts_at_initial_keyframe() { - let source = FakeEncodedSource::new([ - access_unit(1, EncodedFrameType::Delta), - access_unit(2, EncodedFrameType::Delta), - access_unit(3, EncodedFrameType::Key), - access_unit(4, EncodedFrameType::Delta), - ]); - let stats = EncodedVideoPump::new(source).run().unwrap(); - assert_eq!(stats.frames_captured, 2); - } - - #[test] - fn encoded_pump_rejects_empty_payloads() { - let mut unit = access_unit(1, EncodedFrameType::Key); - unit.payload = Bytes::new(); - - let result = EncodedVideoPump::new(FakeEncodedSource::new([unit])).run(); - assert!(matches!(result, Err(PumpError::Capture(CaptureError::EmptyPayload)))); - } - - #[test] - fn encoded_publish_options_use_passthrough() { - let pump = EncodedVideoPump::new(FakeEncodedSource::new([])); - let options = pump.publish_options(); - assert_eq!(options.video_encoder, VideoEncoderBackend::PreEncoded); - assert!(!options.simulcast); - } -} diff --git a/livekit-capture/src/source.rs b/livekit-capture/src/source.rs deleted file mode 100644 index 342306010..000000000 --- a/livekit-capture/src/source.rs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Video capture source traits and types. -//! -//! Everything in this module is independent of libwebrtc: sources produce -//! crate-owned frame types and receive crate-owned feedback types. The pumps -//! in [`pump`](crate::pump) bridge a source into an RTC track and mediate all -//! communication with the WebRTC stack. -//! -//! Both source traits are object-safe, and `Box` boxes implement -//! them, so applications that construct sources dynamically can drive a -//! [`PixelVideoPump>`](crate::pump::PixelVideoPump) while -//! applications that know their source statically pay for no type erasure. - -use std::{error::Error, fmt}; - -use bytes::Bytes; - -use crate::{ - encoded::{EncodedVideoCodec, OwnedEncodedAccessUnit}, - primitive::VideoResolution, -}; - -/// Encoder rate-control target forwarded from WebRTC to an encoded source. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct RateControl { - /// Target bitrate in bits per second. - pub target_bitrate_bps: u64, - /// Target frame rate in frames per second. - pub framerate_fps: f64, -} - -/// Error returned by a capture source. -/// -/// Backend-specific errors are type-erased so sources stay usable as trait -/// objects; the wrapped error remains reachable for display and through -/// [`Error::source`]. -#[derive(Debug)] -pub struct SourceError(Box); - -impl SourceError { - /// Wraps a backend error. - pub fn new(error: impl Into>) -> Self { - Self(error.into()) - } -} - -impl fmt::Display for SourceError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) - } -} - -impl Error for SourceError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - self.0.source() - } -} - -/// Pixel data of one video frame. -#[derive(Debug, Clone)] -#[non_exhaustive] -pub enum PixelVideoData { - /// Planar YUV 4:2:0 with 8-bit samples. - I420 { - /// Luma plane. - y: Bytes, - /// Blue-difference chroma plane. - u: Bytes, - /// Red-difference chroma plane. - v: Bytes, - /// Luma plane stride in bytes. - stride_y: u32, - /// U plane stride in bytes. - stride_u: u32, - /// V plane stride in bytes. - stride_v: u32, - }, -} - -/// One pixel video frame produced by a [`PixelVideoSource`]. -#[derive(Debug, Clone)] -pub struct PixelVideoFrame { - /// Frame resolution in pixels. - pub resolution: VideoResolution, - /// Capture timestamp in microseconds. - pub timestamp_us: i64, - /// Pixel data. - pub data: PixelVideoData, -} - -/// Source of pixel (unencoded) video frames, such as a camera device. -pub trait PixelVideoSource: Send { - /// Nominal output resolution, used to size the RTC source. - fn resolution(&self) -> VideoResolution; - - /// Blocks until the next frame is available, returning `Ok(None)` when - /// the source reaches the end of its stream. - fn next_frame(&mut self) -> Result, SourceError>; -} - -/// Source of pre-encoded video access units, such as an encoding pipeline. -pub trait EncodedVideoSource: Send { - /// Nominal output resolution, used to size the RTC source. - fn resolution(&self) -> VideoResolution; - - /// Codec produced by this source; fixed for the source's lifetime. - fn codec(&self) -> EncodedVideoCodec; - - /// Blocks until the next access unit is available, returning `Ok(None)` - /// when the source reaches the end of its stream. - fn next_access_unit(&mut self) -> Result, SourceError>; - - /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to - /// the producer so it can emit an IDR. - /// - /// The default implementation does nothing, for transports that cannot - /// influence the upstream encoder. - fn request_keyframe(&mut self) {} - - /// Forwards a downstream rate-control target to the producer. - /// - /// The default implementation does nothing, for transports that cannot - /// influence the upstream encoder. - fn update_rate_control(&mut self, _target: RateControl) {} -} - -impl PixelVideoSource for Box { - fn resolution(&self) -> VideoResolution { - (**self).resolution() - } - - fn next_frame(&mut self) -> Result, SourceError> { - (**self).next_frame() - } -} - -impl EncodedVideoSource for Box { - fn resolution(&self) -> VideoResolution { - (**self).resolution() - } - - fn codec(&self) -> EncodedVideoCodec { - (**self).codec() - } - - fn next_access_unit(&mut self) -> Result, SourceError> { - (**self).next_access_unit() - } - - fn request_keyframe(&mut self) { - (**self).request_keyframe() - } - - fn update_rate_control(&mut self, target: RateControl) { - (**self).update_rate_control(target) - } -} - -// Object safety is part of these traits' contract: dynamic applications box -// sources at their edge and drive them through the same generic pumps. -const _: () = { - fn _assert_object_safe(_: &dyn PixelVideoSource, _: &dyn EncodedVideoSource) {} -}; diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index c2a3b630d..d561a89a8 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -22,8 +22,9 @@ use std::{ use bytes::Bytes; use crate::{ + error::SourceError, + pixel::{PixelVideoData, PixelVideoFrame, PixelVideoSource}, primitive::VideoResolution, - source::{PixelVideoData, PixelVideoFrame, PixelVideoSource, SourceError}, }; /// Colors the demo source cycles through, as `(r, g, b)`. diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index ceb3c26ae..0314e0ec0 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -23,11 +23,11 @@ use gst::prelude::*; use crate::{ encoded::{ h26x::{access_unit_from_annex_b, access_unit_from_h264_avc}, - CodecSpecific, EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, + CodecSpecific, EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, + OwnedEncodedAccessUnit, RateControl, }, - error::CaptureError, + error::{CaptureError, SourceError}, primitive::VideoResolution, - source::{EncodedVideoSource, RateControl, SourceError}, }; /// Encoded sample format expected from a GStreamer appsink. From d63de078c17c1aa2259f5eb29d9c6406a68e6fef Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:05:42 -0700 Subject: [PATCH 21/56] Organization --- livekit-capture/Cargo.toml | 2 +- livekit-capture/src/encoded/h26x.rs | 117 ++++++++++++++++------- livekit-capture/src/encoded/mod.rs | 101 ++++++++++--------- livekit-capture/src/encoded/pump.rs | 18 ++-- livekit-capture/src/error.rs | 4 +- livekit-capture/src/pixel/mod.rs | 24 +++-- livekit-capture/src/pixel/pump.rs | 16 ++-- livekit-capture/src/pump.rs | 4 +- livekit-capture/src/sources/demo.rs | 12 +-- livekit-capture/src/sources/gstreamer.rs | 9 +- 10 files changed, 181 insertions(+), 126 deletions(-) diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index d6b3f6e69..7f2a3bd29 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -20,6 +20,6 @@ tokio = { workspace = true, features = ["sync"] } tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] -default = [] +default = ["demo", "gstreamer"] demo = [] gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index baa44d017..da26c026c 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -12,10 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::ops::Range; - -use bytes::Bytes; - use crate::{ encoded::{ annex_b_payload, h264_nal_type, h265_nal_type, is_keyframe_nalus, CodecSpecific, @@ -24,6 +20,8 @@ use crate::{ error::CaptureError, primitive::VideoResolution, }; +use bytes::Bytes; +use std::ops::Range; /// Upper bound on bytes buffered while waiting for an access-unit boundary. const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; @@ -313,13 +311,8 @@ impl AvcAccessUnitParser { self.scan_cursor -= byte_len; let timestamp_us = self.next_timestamp_us; self.next_timestamp_us = self.next_timestamp_us.saturating_add(self.frame_interval_us); - access_unit_from_h264_avc( - &access_unit, - self.nal_length_size, - timestamp_us, - self.resolution, - ) - .map(Some) + access_unit_from_h264_avc(&access_unit, self.nal_length_size, timestamp_us, self.resolution) + .map(Some) } } @@ -637,15 +630,22 @@ mod tests { #[test] fn access_unit_from_avc_rejects_truncated_nal() { - let err = access_unit_from_h264_avc(&[0, 0, 0, 3, 0x65], 4, 10, VideoResolution::new(640, 480)).unwrap_err(); + let err = + access_unit_from_h264_avc(&[0, 0, 0, 3, 0x65], 4, 10, VideoResolution::new(640, 480)) + .unwrap_err(); assert_eq!(err, CaptureError::InvalidEncodedData("truncated AVC NAL unit")); } #[test] fn parser_flushes_final_access_unit() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 100, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); assert!(parser.push(&[0, 0, 1, 0x65, 1, 2]).unwrap().is_none()); let au = parser.flush().unwrap().unwrap(); assert_eq!(au.timestamp_us, 100); @@ -654,8 +654,13 @@ mod tests { #[test] fn parser_splits_at_next_access_unit_delimiter() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 100, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); let stream = [0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x65, 1, 2, 0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x41, 3]; @@ -670,7 +675,8 @@ mod tests { #[test] fn avc_parser_splits_at_next_access_unit_delimiter() { - let mut parser = AvcAccessUnitParser::new(4, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = + AvcAccessUnitParser::new(4, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); let stream = [ 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 3, 0x65, 1, 2, 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 2, 0x41, 3, @@ -687,8 +693,13 @@ mod tests { #[test] fn splits_aud_less_h264_stream_per_frame() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); let stream = [ 0, 0, 0, 1, 0x67, 0x42, 0x00, 0x1e, // SPS 0, 0, 0, 1, 0x68, 0xce, // PPS @@ -714,8 +725,13 @@ mod tests { #[test] fn keeps_multi_slice_h264_access_unit_together() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); let stream = [ 0, 0, 1, 0x65, 0x88, 0x11, // IDR slice, first_mb_in_slice == 0 0, 0, 1, 0x65, 0x21, 0x22, // IDR slice, first_mb_in_slice != 0 @@ -734,8 +750,13 @@ mod tests { #[test] fn splits_aud_less_h265_stream_per_frame() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H265, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); let stream = [ 0, 0, 0, 1, 0x40, 0x01, 0x0c, // VPS 0, 0, 0, 1, 0x42, 0x01, 0x02, // SPS @@ -758,8 +779,13 @@ mod tests { #[test] fn keeps_multi_slice_h265_access_unit_together() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H265, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); let stream = [ 0, 0, 1, 0x26, 0x01, 0xaf, 0x11, // IDR slice, first_slice_segment_in_pic_flag == 1 @@ -780,8 +806,13 @@ mod tests { #[test] fn groups_parameter_sets_with_following_frame() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); let stream = [ 0, 0, 1, 0x67, 0x42, 0x1e, // SPS 0, 0, 1, 0x68, 0xce, // PPS @@ -847,7 +878,15 @@ mod tests { 0, 0, 0, 1, 0x41, 0x9a, 0x04, 0x00, // P, first_mb_in_slice == 0 ]; assert_chunked_matches_one_shot( - || AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(), + || { + AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap() + }, &h264_annex_b, 4, ); @@ -862,7 +901,15 @@ mod tests { 0, 0, 1, 0x02, 0x01, 0xd0, 0x0a, // TRAIL_R ]; assert_chunked_matches_one_shot( - || AnnexBAccessUnitParser::new(EncodedVideoCodec::H265, 0, 33_333, VideoResolution::new(640, 480)).unwrap(), + || { + AnnexBAccessUnitParser::new( + EncodedVideoCodec::H265, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap() + }, &h265_annex_b, 3, ); @@ -884,8 +931,13 @@ mod tests { #[test] fn rejects_pending_access_unit_over_size_cap() { - let mut parser = - AnnexBAccessUnitParser::new(EncodedVideoCodec::H264, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); assert!(parser.push(&[0, 0, 1, 0x65, 0x88]).unwrap().is_none()); let err = parser.push(&vec![0xff; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); @@ -897,7 +949,8 @@ mod tests { #[test] fn avc_rejects_pending_access_unit_over_size_cap() { - let mut parser = AvcAccessUnitParser::new(4, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let mut parser = + AvcAccessUnitParser::new(4, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); let nal_len = (MAX_PENDING_ACCESS_UNIT_BYTES + 1) as u32; assert!(parser.push(&nal_len.to_be_bytes()).unwrap().is_none()); diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index bbe9e423a..5fe21cc72 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -21,12 +21,11 @@ //! implemented for `Box`, so sources can be constructed dynamically //! and driven through the same generic pump. -pub mod h26x; -mod pump; - +use crate::{ + error::{CaptureError, SourceError}, + primitive::VideoResolution, +}; use bytes::Bytes; - -pub use pump::EncodedVideoPump; use livekit::{ options::VideoCodec, webrtc::video_frame::{ @@ -34,13 +33,38 @@ use livekit::{ }, }; -use crate::{ - error::{CaptureError, SourceError}, - primitive::VideoResolution, -}; +pub mod h26x; +mod pump; +pub use pump::EncodedVideoPump; const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; +/// Source of pre-encoded video access units, such as an encoding pipeline. +pub trait EncodedVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Codec produced by this source; fixed for the source's lifetime. + fn codec(&self) -> EncodedVideoCodec; + + /// Blocks until the next access unit is available, returning `Ok(None)` + /// when the source reaches the end of its stream. + fn next_access_unit(&mut self) -> Result, SourceError>; + + /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to + /// the producer so it can emit an IDR. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn request_keyframe(&mut self) {} + + /// Forwards a downstream rate-control target to the producer. + /// + /// The default implementation does nothing, for transports that cannot + /// influence the upstream encoder. + fn update_rate_control(&mut self, _target: RateControl) {} +} + /// Encoded byte-stream framing used by encoded source backends. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] @@ -435,32 +459,6 @@ pub struct RateControl { pub framerate_fps: f64, } -/// Source of pre-encoded video access units, such as an encoding pipeline. -pub trait EncodedVideoSource: Send { - /// Nominal output resolution, used to size the RTC source. - fn resolution(&self) -> VideoResolution; - - /// Codec produced by this source; fixed for the source's lifetime. - fn codec(&self) -> EncodedVideoCodec; - - /// Blocks until the next access unit is available, returning `Ok(None)` - /// when the source reaches the end of its stream. - fn next_access_unit(&mut self) -> Result, SourceError>; - - /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to - /// the producer so it can emit an IDR. - /// - /// The default implementation does nothing, for transports that cannot - /// influence the upstream encoder. - fn request_keyframe(&mut self) {} - - /// Forwards a downstream rate-control target to the producer. - /// - /// The default implementation does nothing, for transports that cannot - /// influence the upstream encoder. - fn update_rate_control(&mut self, _target: RateControl) {} -} - impl EncodedVideoSource for Box { fn resolution(&self) -> VideoResolution { (**self).resolution() @@ -532,7 +530,9 @@ mod tests { fn h264_nal_helper_assembles_annex_b_and_detects_keyframe() { let sps = [0x67, 1, 2, 3]; let idr = [0x65, 4, 5, 6]; - let au = EncodedAccessUnit::from_h264_nalus(&[&sps, &idr], 10, VideoResolution::new(640, 480)).unwrap(); + let au = + EncodedAccessUnit::from_h264_nalus(&[&sps, &idr], 10, VideoResolution::new(640, 480)) + .unwrap(); assert_eq!(au.codec, EncodedVideoCodec::H264); assert_eq!(au.frame_type, EncodedFrameType::Key); @@ -548,14 +548,25 @@ mod tests { let sps = [0x42, 1, 2]; let pps = [0x44, 1, 2]; let idr_w_radl = [19 << 1, 1, 3]; - let idr_without_headers = - EncodedAccessUnit::from_h265_nalus(&[&vps, &idr_w_radl], 10, VideoResolution::new(640, 480)).unwrap(); - let key = - EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &idr_w_radl], 10, VideoResolution::new(640, 480)) - .unwrap(); + let idr_without_headers = EncodedAccessUnit::from_h265_nalus( + &[&vps, &idr_w_radl], + 10, + VideoResolution::new(640, 480), + ) + .unwrap(); + let key = EncodedAccessUnit::from_h265_nalus( + &[&vps, &sps, &pps, &idr_w_radl], + 10, + VideoResolution::new(640, 480), + ) + .unwrap(); let cra = [21 << 1, 1, 3]; - let cra_with_headers = - EncodedAccessUnit::from_h265_nalus(&[&vps, &sps, &pps, &cra], 10, VideoResolution::new(640, 480)).unwrap(); + let cra_with_headers = EncodedAccessUnit::from_h265_nalus( + &[&vps, &sps, &pps, &cra], + 10, + VideoResolution::new(640, 480), + ) + .unwrap(); assert_eq!(idr_without_headers.codec, EncodedVideoCodec::H265); assert_eq!(idr_without_headers.frame_type, EncodedFrameType::Delta); @@ -565,7 +576,9 @@ mod tests { #[test] fn h265_rejects_too_short_nal_header() { - let err = EncodedAccessUnit::from_h265_nalus(&[&[0x26]], 10, VideoResolution::new(640, 480)).unwrap_err(); + let err = + EncodedAccessUnit::from_h265_nalus(&[&[0x26]], 10, VideoResolution::new(640, 480)) + .unwrap_err(); assert_eq!(err, CaptureError::H265NalTooShort); } diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs index 4f360d29c..ec98ed4de 100644 --- a/livekit-capture/src/encoded/pump.rs +++ b/livekit-capture/src/encoded/pump.rs @@ -15,16 +15,6 @@ //! Pumps encoded access units from a capture source into an RTC video //! source. -use std::{fmt, io}; - -use livekit::{ - options::{TrackPublishOptions, VideoEncoderBackend}, - webrtc::{ - video_frame::EncodedVideoFrame, - video_source::{native::NativeVideoSource, EncodedRateControl, RtcVideoSource}, - }, -}; - use crate::{ encoded::{ CodecSpecific, EncodedFrameType, EncodedLayerInfo, EncodedVideoSource, @@ -33,6 +23,14 @@ use crate::{ error::CaptureError, pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, }; +use livekit::{ + options::{TrackPublishOptions, VideoEncoderBackend}, + webrtc::{ + video_frame::EncodedVideoFrame, + video_source::{native::NativeVideoSource, EncodedRateControl, RtcVideoSource}, + }, +}; +use std::{fmt, io}; impl From for RateControl { fn from(target: EncodedRateControl) -> Self { diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index ee6d2752b..acb9f2ab0 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -14,12 +14,10 @@ //! Error types shared across capture paths. +use crate::encoded::{EncodedVideoCodec, EncodedWireFormat}; use std::{error::Error as StdError, fmt}; - use thiserror::Error; -use crate::encoded::{EncodedVideoCodec, EncodedWireFormat}; - /// Error returned by a capture source. /// /// Backend-specific errors are type-erased so sources stay usable as trait diff --git a/livekit-capture/src/pixel/mod.rs b/livekit-capture/src/pixel/mod.rs index d1c313289..b4d037301 100644 --- a/livekit-capture/src/pixel/mod.rs +++ b/livekit-capture/src/pixel/mod.rs @@ -19,13 +19,21 @@ //! object-safe and implemented for `Box`, so sources can be //! constructed dynamically and driven through the same generic pump. -mod pump; - +use crate::{error::SourceError, primitive::VideoResolution}; use bytes::Bytes; +mod pump; pub use pump::PixelVideoPump; -use crate::{error::SourceError, primitive::VideoResolution}; +/// Source of pixel (unencoded) video frames, such as a camera device. +pub trait PixelVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Blocks until the next frame is available, returning `Ok(None)` when + /// the source reaches the end of its stream. + fn next_frame(&mut self) -> Result, SourceError>; +} /// Pixel data of one video frame. #[derive(Debug, Clone)] @@ -59,16 +67,6 @@ pub struct PixelVideoFrame { pub data: PixelVideoData, } -/// Source of pixel (unencoded) video frames, such as a camera device. -pub trait PixelVideoSource: Send { - /// Nominal output resolution, used to size the RTC source. - fn resolution(&self) -> VideoResolution; - - /// Blocks until the next frame is available, returning `Ok(None)` when - /// the source reaches the end of its stream. - fn next_frame(&mut self) -> Result, SourceError>; -} - impl PixelVideoSource for Box { fn resolution(&self) -> VideoResolution { (**self).resolution() diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs index 6e662ebcf..78eaeca6c 100644 --- a/livekit-capture/src/pixel/pump.rs +++ b/livekit-capture/src/pixel/pump.rs @@ -14,8 +14,12 @@ //! Pumps pixel frames from a capture source into an RTC video source. -use std::{fmt, io}; - +use crate::{ + error::CaptureError, + pixel::{PixelVideoData, PixelVideoFrame, PixelVideoSource}, + primitive::VideoResolution, + pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, +}; use livekit::{ options::TrackPublishOptions, webrtc::{ @@ -23,13 +27,7 @@ use livekit::{ video_source::{native::NativeVideoSource, RtcVideoSource}, }, }; - -use crate::{ - error::CaptureError, - pixel::{PixelVideoData, PixelVideoFrame, PixelVideoSource}, - primitive::VideoResolution, - pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, -}; +use std::{fmt, io}; /// Pumps a [`PixelVideoSource`] into an RTC video source, publishing frames /// through the WebRTC encoder. diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index 31ac468f5..6ce950238 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -23,6 +23,7 @@ //! spawn into the same [`RunningPump`] defined here, so running pumps of //! either kind are supervised uniformly. +use crate::error::{CaptureError, SourceError}; use std::{ any::Any, io, @@ -33,11 +34,8 @@ use std::{ }, thread, }; - use thiserror::Error; -use crate::error::{CaptureError, SourceError}; - /// Error returned by a pump run. #[derive(Debug, Error)] pub enum PumpError { diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index d561a89a8..3e179130f 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -14,18 +14,16 @@ //! Solid-color demo source for testing. -use std::{ - thread, - time::{Duration, Instant}, -}; - -use bytes::Bytes; - use crate::{ error::SourceError, pixel::{PixelVideoData, PixelVideoFrame, PixelVideoSource}, primitive::VideoResolution, }; +use bytes::Bytes; +use std::{ + thread, + time::{Duration, Instant}, +}; /// Colors the demo source cycles through, as `(r, g, b)`. const PALETTE: [(u8, u8, u8); 6] = [ diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 0314e0ec0..1c7296327 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use bytes::Bytes; -use thiserror::Error; - use ::gstreamer as gst; use ::gstreamer_app as gst_app; +use bytes::Bytes; use gst::glib; use gst::prelude::*; +use thiserror::Error; use crate::{ encoded::{ @@ -244,7 +243,9 @@ impl EncodedVideoSource for GStreamerVideoSource { match self.appsink.pull_sample() { Ok(sample) => self.access_unit_from_sample(&sample).map(Some).map_err(SourceError::new), Err(_err) if self.appsink.is_eos() => Ok(None), - Err(err) => Err(SourceError::new(GStreamerVideoSourceError::PullSample(err.to_string()))), + Err(err) => { + Err(SourceError::new(GStreamerVideoSourceError::PullSample(err.to_string()))) + } } } From e59e34877d28dfefa2ddc5a3d6cd938ec1b88aae Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:22:47 -0700 Subject: [PATCH 22/56] Remove source mut methods --- livekit-capture/src/encoded/pump.rs | 5 ----- livekit-capture/src/pixel/pump.rs | 5 ----- 2 files changed, 10 deletions(-) diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs index ec98ed4de..6a055c147 100644 --- a/livekit-capture/src/encoded/pump.rs +++ b/livekit-capture/src/encoded/pump.rs @@ -84,11 +84,6 @@ impl EncodedVideoPump { &self.source } - /// Returns the underlying capture source mutably. - pub fn source_mut(&mut self) -> &mut S { - &mut self.source - } - /// Runs the pump on the calling thread until the source ends, a failure, /// or the stop handle fires. /// diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs index 78eaeca6c..b30c5c794 100644 --- a/livekit-capture/src/pixel/pump.rs +++ b/livekit-capture/src/pixel/pump.rs @@ -68,11 +68,6 @@ impl PixelVideoPump { &self.source } - /// Returns the underlying capture source mutably. - pub fn source_mut(&mut self) -> &mut S { - &mut self.source - } - /// Runs the pump on the calling thread until the source ends, a failure, /// or the stop handle fires. /// From 61278a52e39ddb432dba651af09690de3ea83f7b Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:23:36 -0700 Subject: [PATCH 23/56] Add TODO --- livekit-capture/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 7f2a3bd29..a31db41a6 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -20,6 +20,6 @@ tokio = { workspace = true, features = ["sync"] } tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] -default = ["demo", "gstreamer"] +default = ["demo", "gstreamer"] # TODO: Remove after testing demo = [] gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] From 94e21acff44ee65999927914e07fc8d00a279854 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:27:15 -0700 Subject: [PATCH 24/56] Sort features --- livekit-capture/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index a31db41a6..810a0dfe6 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -21,5 +21,9 @@ tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] default = ["demo", "gstreamer"] # TODO: Remove after testing + +# Pixel sources demo = [] + +# Encoded sources gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] From 7bb98ee63c3416d492b90cca18d9d969353b4c48 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:53:20 -0700 Subject: [PATCH 25/56] Use webrtc frames --- livekit-capture/README.md | 14 +- livekit-capture/src/encoded/mod.rs | 26 ++-- livekit-capture/src/encoded/pump.rs | 59 +++++++-- livekit-capture/src/error.rs | 3 - livekit-capture/src/pixel/mod.rs | 56 +++----- livekit-capture/src/pixel/pump.rs | 160 +++++++++-------------- livekit-capture/src/sources/demo.rs | 65 ++++----- livekit-capture/src/sources/gstreamer.rs | 7 +- 8 files changed, 171 insertions(+), 219 deletions(-) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index b07712ac3..a1c9b93a6 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -6,13 +6,15 @@ ingest source; the `demo` feature adds a synthetic pixel source for testing. ## Library entry points -- `pixel::PixelVideoSource` and `encoded::EncodedVideoSource` — the - libwebrtc-free traits a capture backend implements: pixel sources produce - frames published through the WebRTC encoder, encoded sources produce - access units published as passthrough. Both traits are object-safe and +- `pixel::PixelVideoSource` and `encoded::EncodedVideoSource` — the traits a + capture backend implements: pixel sources yield libwebrtc `VideoFrame`s + (any `VideoBuffer`, CPU or native, with no intermediate copy) published + through the WebRTC encoder; encoded sources produce crate-owned access + units published as passthrough. Both traits are object-safe and implemented for `Box`, so sources can be constructed dynamically - and driven through the same pumps. Each kind module also holds that kind's - vocabulary (`pixel::PixelVideoFrame`, `encoded::EncodedAccessUnit`, …). + and driven through the same pumps. The crate owns a type only where it + adds semantics (`encoded::EncodedAccessUnit` and the parsing/validation + vocabulary); elsewhere livekit's types are used directly. - `pixel::PixelVideoPump` and `encoded::EncodedVideoPump` — bridge a source into a publishable RTC track: each builds the matching `NativeVideoSource`, derives publish options (`EncodedVideoPump` selects diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index 5fe21cc72..84de9bcfd 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -15,9 +15,9 @@ //! Encoded video: codec vocabulary, access units, the source contract for //! pre-encoded ingest, and the pump. //! -//! Sources produce crate-owned access units independent of libwebrtc and -//! receive crate-owned feedback types, and [`EncodedVideoPump`] bridges them -//! into an RTC track as passthrough. The source trait is object-safe and +//! Sources produce crate-owned access units — the vocabulary the parsing +//! and validation helpers speak — and [`EncodedVideoPump`] bridges them into +//! an RTC track as passthrough. The source trait is object-safe and //! implemented for `Box`, so sources can be constructed dynamically //! and driven through the same generic pump. @@ -28,8 +28,11 @@ use crate::{ use bytes::Bytes; use livekit::{ options::VideoCodec, - webrtc::video_frame::{ - EncodedFrameType as RtcEncodedFrameType, EncodedVideoCodec as RtcEncodedVideoCodec, + webrtc::{ + video_frame::{ + EncodedFrameType as RtcEncodedFrameType, EncodedVideoCodec as RtcEncodedVideoCodec, + }, + video_source::EncodedRateControl, }, }; @@ -62,7 +65,7 @@ pub trait EncodedVideoSource: Send { /// /// The default implementation does nothing, for transports that cannot /// influence the upstream encoder. - fn update_rate_control(&mut self, _target: RateControl) {} + fn update_rate_control(&mut self, _target: EncodedRateControl) {} } /// Encoded byte-stream framing used by encoded source backends. @@ -450,15 +453,6 @@ impl From for RtcEncodedFrameType { } } -/// Encoder rate-control target forwarded from WebRTC to an encoded source. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct RateControl { - /// Target bitrate in bits per second. - pub target_bitrate_bps: u64, - /// Target frame rate in frames per second. - pub framerate_fps: f64, -} - impl EncodedVideoSource for Box { fn resolution(&self) -> VideoResolution { (**self).resolution() @@ -476,7 +470,7 @@ impl EncodedVideoSource for Box { (**self).request_keyframe() } - fn update_rate_control(&mut self, target: RateControl) { + fn update_rate_control(&mut self, target: EncodedRateControl) { (**self).update_rate_control(target) } } diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs index 6a055c147..19228df72 100644 --- a/livekit-capture/src/encoded/pump.rs +++ b/livekit-capture/src/encoded/pump.rs @@ -18,7 +18,7 @@ use crate::{ encoded::{ CodecSpecific, EncodedFrameType, EncodedLayerInfo, EncodedVideoSource, - OwnedEncodedAccessUnit, RateControl, + OwnedEncodedAccessUnit, }, error::CaptureError, pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, @@ -26,17 +26,14 @@ use crate::{ use livekit::{ options::{TrackPublishOptions, VideoEncoderBackend}, webrtc::{ - video_frame::EncodedVideoFrame, - video_source::{native::NativeVideoSource, EncodedRateControl, RtcVideoSource}, + video_frame::{EncodedVideoFrame, FrameMetadata}, + video_source::{native::NativeVideoSource, RtcVideoSource}, }, }; use std::{fmt, io}; -impl From for RateControl { - fn from(target: EncodedRateControl) -> Self { - Self { target_bitrate_bps: target.target_bitrate_bps, framerate_fps: target.framerate_fps } - } -} +/// Callback that supplies packet-trailer metadata for an access unit. +type FrameMetadataFn = Box Option + Send>; /// Pumps an [`EncodedVideoSource`] into an RTC video source, publishing /// access units as passthrough. @@ -49,6 +46,7 @@ pub struct EncodedVideoPump { source: S, rtc_source: NativeVideoSource, stop: PumpStop, + frame_metadata: Option, } impl EncodedVideoPump { @@ -56,7 +54,21 @@ impl EncodedVideoPump { /// source. pub fn new(source: S) -> Self { let rtc_source = NativeVideoSource::new_encoded(source.resolution().into()); - Self { source, rtc_source, stop: PumpStop::new() } + Self { source, rtc_source, stop: PumpStop::new(), frame_metadata: None } + } + + /// Sets a callback that supplies packet-trailer metadata for each access + /// unit before it is captured. + /// + /// Metadata is only propagated to subscribers when the corresponding + /// [`TrackPublishOptions::frame_metadata_features`] are enabled before + /// publishing the local track. + pub fn with_frame_metadata( + mut self, + frame_metadata: impl FnMut(&OwnedEncodedAccessUnit) -> Option + Send + 'static, + ) -> Self { + self.frame_metadata = Some(Box::new(frame_metadata)); + self } /// Returns the RTC source to create the local track with. @@ -97,7 +109,7 @@ impl EncodedVideoPump { break PumpExit::Stopped; } if let Some(target) = self.rtc_source.take_rate_control_request() { - self.source.update_rate_control(target.into()); + self.source.update_rate_control(target); } if self.rtc_source.take_keyframe_request() { self.source.request_keyframe(); @@ -113,7 +125,8 @@ impl EncodedVideoPump { } awaiting_initial_keyframe = false; - capture_access_unit(&self.rtc_source, &access_unit)?; + let metadata = self.frame_metadata.as_mut().and_then(|callback| callback(&access_unit)); + capture_access_unit(&self.rtc_source, &access_unit, metadata)?; frames_captured += 1; }; Ok(PumpStats { frames_captured, exit }) @@ -144,6 +157,7 @@ impl fmt::Debug for EncodedVideoPump { fn capture_access_unit( rtc_source: &NativeVideoSource, access_unit: &OwnedEncodedAccessUnit, + frame_metadata: Option, ) -> Result<(), CaptureError> { validate_access_unit(access_unit)?; @@ -153,7 +167,7 @@ fn capture_access_unit( timestamp_us: access_unit.timestamp_us, frame_type: access_unit.frame_type.into(), resolution: access_unit.resolution.into(), - frame_metadata: None, + frame_metadata, }; rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) } @@ -248,6 +262,27 @@ mod tests { assert_eq!(stats.frames_captured, 1); } + #[test] + fn metadata_callback_runs_per_captured_access_unit() { + let source = FakeEncodedSource::new([ + access_unit(1, EncodedFrameType::Delta), // dropped pre-roll, no callback + access_unit(2, EncodedFrameType::Key), + access_unit(3, EncodedFrameType::Delta), + ]); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let calls_in_callback = calls.clone(); + let stats = EncodedVideoPump::new(source) + .with_frame_metadata(move |access_unit| { + assert!(access_unit.timestamp_us > 1); + calls_in_callback.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + None + }) + .run() + .unwrap(); + assert_eq!(stats.frames_captured, 2); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 2); + } + #[test] fn encoded_pump_rejects_empty_payloads() { let mut unit = access_unit(1, EncodedFrameType::Key); diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index acb9f2ab0..c72255817 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -63,9 +63,6 @@ pub enum CaptureError { /// Encoded payload or transport data is malformed. #[error("invalid encoded data: {0}")] InvalidEncodedData(&'static str), - /// Pixel frame data is malformed. - #[error("invalid pixel frame: {0}")] - InvalidPixelFrame(&'static str), /// Wire format is represented by the API but not supported by this source. #[error("encoded wire format is not supported by this source: {0:?}")] UnsupportedWireFormat(EncodedWireFormat), diff --git a/livekit-capture/src/pixel/mod.rs b/livekit-capture/src/pixel/mod.rs index b4d037301..e64e1fdde 100644 --- a/livekit-capture/src/pixel/mod.rs +++ b/livekit-capture/src/pixel/mod.rs @@ -12,19 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Pixel (unencoded) video: frame types, the source contract, and the pump. +//! Pixel (unencoded) video: the source contract and the pump. //! -//! Sources produce crate-owned frame types independent of libwebrtc, and -//! [`PixelVideoPump`] bridges them into an RTC track. The source trait is +//! Sources yield libwebrtc [`VideoFrame`](livekit::webrtc::video_frame::VideoFrame)s +//! directly, so any [`VideoBuffer`](livekit::webrtc::video_frame::VideoBuffer) +//! implementation — CPU planes or platform-native — reaches the RTC track +//! without an intermediate copy. [`PixelVideoPump`] drives a source and +//! publishes its frames through the WebRTC encoder. The source trait is //! object-safe and implemented for `Box`, so sources can be //! constructed dynamically and driven through the same generic pump. -use crate::{error::SourceError, primitive::VideoResolution}; -use bytes::Bytes; - mod pump; + +use livekit::webrtc::video_frame::BoxVideoFrame; + pub use pump::PixelVideoPump; +use crate::{error::SourceError, primitive::VideoResolution}; + /// Source of pixel (unencoded) video frames, such as a camera device. pub trait PixelVideoSource: Send { /// Nominal output resolution, used to size the RTC source. @@ -32,39 +37,10 @@ pub trait PixelVideoSource: Send { /// Blocks until the next frame is available, returning `Ok(None)` when /// the source reaches the end of its stream. - fn next_frame(&mut self) -> Result, SourceError>; -} - -/// Pixel data of one video frame. -#[derive(Debug, Clone)] -#[non_exhaustive] -pub enum PixelVideoData { - /// Planar YUV 4:2:0 with 8-bit samples. - I420 { - /// Luma plane. - y: Bytes, - /// Blue-difference chroma plane. - u: Bytes, - /// Red-difference chroma plane. - v: Bytes, - /// Luma plane stride in bytes. - stride_y: u32, - /// U plane stride in bytes. - stride_u: u32, - /// V plane stride in bytes. - stride_v: u32, - }, -} - -/// One pixel video frame produced by a [`PixelVideoSource`]. -#[derive(Debug, Clone)] -pub struct PixelVideoFrame { - /// Frame resolution in pixels. - pub resolution: VideoResolution, - /// Capture timestamp in microseconds. - pub timestamp_us: i64, - /// Pixel data. - pub data: PixelVideoData, + /// + /// Sources may pre-fill the frame's `frame_metadata`; a metadata + /// callback set on the pump takes precedence when it returns `Some`. + fn next_frame(&mut self) -> Result, SourceError>; } impl PixelVideoSource for Box { @@ -72,7 +48,7 @@ impl PixelVideoSource for Box { (**self).resolution() } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self) -> Result, SourceError> { (**self).next_frame() } } diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs index b30c5c794..d618a47ec 100644 --- a/livekit-capture/src/pixel/pump.rs +++ b/livekit-capture/src/pixel/pump.rs @@ -15,26 +15,28 @@ //! Pumps pixel frames from a capture source into an RTC video source. use crate::{ - error::CaptureError, - pixel::{PixelVideoData, PixelVideoFrame, PixelVideoSource}, - primitive::VideoResolution, + pixel::PixelVideoSource, pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, }; use livekit::{ options::TrackPublishOptions, webrtc::{ - video_frame::{I420Buffer, VideoFrame, VideoRotation}, + video_frame::{BoxVideoFrame, FrameMetadata}, video_source::{native::NativeVideoSource, RtcVideoSource}, }, }; use std::{fmt, io}; +/// Callback that supplies packet-trailer metadata for a pixel frame. +type FrameMetadataFn = Box Option + Send>; + /// Pumps a [`PixelVideoSource`] into an RTC video source, publishing frames /// through the WebRTC encoder. pub struct PixelVideoPump { source: S, rtc_source: NativeVideoSource, stop: PumpStop, + frame_metadata: Option, } impl PixelVideoPump { @@ -45,7 +47,23 @@ impl PixelVideoPump { /// The pump itself runs on plain threads. pub fn new(source: S) -> Self { let rtc_source = NativeVideoSource::new(source.resolution().into(), false); - Self { source, rtc_source, stop: PumpStop::new() } + Self { source, rtc_source, stop: PumpStop::new(), frame_metadata: None } + } + + /// Sets a callback that supplies packet-trailer metadata for each frame + /// before it is captured. + /// + /// When the callback returns `Some`, it overrides any metadata the + /// source pre-filled on the frame. Metadata is only propagated to + /// subscribers when the corresponding + /// [`TrackPublishOptions::frame_metadata_features`] are enabled before + /// publishing the local track. + pub fn with_frame_metadata( + mut self, + frame_metadata: impl FnMut(&BoxVideoFrame) -> Option + Send + 'static, + ) -> Self { + self.frame_metadata = Some(Box::new(frame_metadata)); + self } /// Returns the RTC source to create the local track with. @@ -79,10 +97,15 @@ impl PixelVideoPump { if self.stop.is_stopped() { break PumpExit::Stopped; } - let Some(frame) = self.source.next_frame()? else { + let Some(mut frame) = self.source.next_frame()? else { break PumpExit::EndOfStream; }; - capture_pixel_frame(&self.rtc_source, &frame)?; + if let Some(metadata) = + self.frame_metadata.as_mut().and_then(|callback| callback(&frame)) + { + frame.frame_metadata = Some(metadata); + } + self.rtc_source.capture_frame(&frame); frames_captured += 1; }; Ok(PumpStats { frames_captured, exit }) @@ -110,70 +133,14 @@ impl fmt::Debug for PixelVideoPump { } } -fn capture_pixel_frame( - rtc_source: &NativeVideoSource, - frame: &PixelVideoFrame, -) -> Result<(), CaptureError> { - let buffer = i420_buffer(frame)?; - rtc_source.capture_frame(&VideoFrame { - rotation: VideoRotation::VideoRotation0, - timestamp_us: frame.timestamp_us, - frame_metadata: None, - buffer, - }); - Ok(()) -} - -fn i420_buffer(frame: &PixelVideoFrame) -> Result { - let PixelVideoData::I420 { y, u, v, stride_y, stride_u, stride_v } = &frame.data; - - let VideoResolution { width, height } = frame.resolution; - let mut buffer = I420Buffer::new(width, height); - let chroma_width = width.div_ceil(2); - let chroma_height = height.div_ceil(2); - let (dst_stride_y, dst_stride_u, dst_stride_v) = buffer.strides(); - let (dst_y, dst_u, dst_v) = buffer.data_mut(); - - copy_plane(y, *stride_y, dst_y, dst_stride_y, width, height)?; - copy_plane(u, *stride_u, dst_u, dst_stride_u, chroma_width, chroma_height)?; - copy_plane(v, *stride_v, dst_v, dst_stride_v, chroma_width, chroma_height)?; - Ok(buffer) -} - -fn copy_plane( - src: &[u8], - src_stride: u32, - dst: &mut [u8], - dst_stride: u32, - width: u32, - height: u32, -) -> Result<(), CaptureError> { - let (width, height) = (width as usize, height as usize); - let (src_stride, dst_stride) = (src_stride as usize, dst_stride as usize); - if src_stride < width { - return Err(CaptureError::InvalidPixelFrame("plane stride is smaller than its width")); - } - // The final row may be unpadded. - let min_len = (height - 1).saturating_mul(src_stride) + width; - if src.len() < min_len { - return Err(CaptureError::InvalidPixelFrame("plane data is shorter than its dimensions")); - } - - for row in 0..height { - let src_row = &src[row * src_stride..][..width]; - dst[row * dst_stride..][..width].copy_from_slice(src_row); - } - Ok(()) -} - #[cfg(test)] mod tests { use std::collections::VecDeque; - use bytes::Bytes; + use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation}; use super::*; - use crate::error::SourceError; + use crate::{error::SourceError, primitive::VideoResolution}; const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; @@ -186,29 +153,21 @@ mod tests { .expect("failed to build test runtime") } - fn pixel_frame(timestamp_us: i64) -> PixelVideoFrame { - let chroma_width = RESOLUTION.width.div_ceil(2); - let chroma_height = RESOLUTION.height.div_ceil(2); - PixelVideoFrame { - resolution: RESOLUTION, + fn pixel_frame(timestamp_us: i64) -> BoxVideoFrame { + VideoFrame { + rotation: VideoRotation::VideoRotation0, timestamp_us, - data: PixelVideoData::I420 { - y: Bytes::from(vec![128; (RESOLUTION.width * RESOLUTION.height) as usize]), - u: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), - v: Bytes::from(vec![128; (chroma_width * chroma_height) as usize]), - stride_y: RESOLUTION.width, - stride_u: chroma_width, - stride_v: chroma_width, - }, + frame_metadata: None, + buffer: Box::new(I420Buffer::new(RESOLUTION.width, RESOLUTION.height)), } } struct FakePixelSource { - frames: VecDeque, + frames: VecDeque, } impl FakePixelSource { - fn new(frames: impl IntoIterator) -> Self { + fn new(frames: impl IntoIterator) -> Self { Self { frames: frames.into_iter().collect() } } } @@ -218,7 +177,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self) -> Result, SourceError> { Ok(self.frames.pop_front()) } } @@ -246,6 +205,26 @@ mod tests { assert_eq!(stats.frames_captured, 2); } + #[test] + fn metadata_callback_runs_per_frame() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let calls_in_callback = calls.clone(); + let stats = PixelVideoPump::new(source) + .with_frame_metadata(move |frame| { + assert!(frame.timestamp_us > 0); + calls_in_callback.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + None + }) + .run() + .unwrap(); + assert_eq!(stats.frames_captured, 3); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 3); + } + #[test] fn pump_panics_become_errors() { struct PanickingSource; @@ -255,7 +234,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self) -> Result, SourceError> { panic!("source exploded"); } } @@ -279,7 +258,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self) -> Result, SourceError> { std::thread::sleep(std::time::Duration::from_millis(1)); Ok(Some(pixel_frame(0))) } @@ -295,19 +274,6 @@ mod tests { assert_eq!(stats.exit, PumpExit::Stopped); } - #[test] - fn pixel_pump_rejects_short_planes() { - let runtime = runtime_context(); - let _guard = runtime.enter(); - - let mut frame = pixel_frame(1); - let PixelVideoData::I420 { y, .. } = &mut frame.data; - *y = Bytes::from(vec![128; 8]); - - let result = PixelVideoPump::new(FakePixelSource::new([frame])).run(); - assert!(matches!(result, Err(PumpError::Capture(CaptureError::InvalidPixelFrame(_))))); - } - #[tokio::test] async fn pump_stops_and_joins_async() { struct EndlessSource; @@ -317,7 +283,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self) -> Result, SourceError> { std::thread::sleep(std::time::Duration::from_millis(1)); Ok(Some(pixel_frame(0))) } diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index 3e179130f..07fffe296 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -14,12 +14,8 @@ //! Solid-color demo source for testing. -use crate::{ - error::SourceError, - pixel::{PixelVideoData, PixelVideoFrame, PixelVideoSource}, - primitive::VideoResolution, -}; -use bytes::Bytes; +use crate::{error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution}; +use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; use std::{ thread, time::{Duration, Instant}, @@ -66,8 +62,8 @@ impl Default for DemoSourceConfig { #[derive(Debug)] pub struct DemoSource { config: DemoSourceConfig, - /// One pre-rendered `(y, u, v)` plane set per palette color. - planes: Vec<(Bytes, Bytes, Bytes)>, + /// One `(y, u, v)` sample triple per palette color. + colors: Vec<(u8, u8, u8)>, started: Option, frame_index: u64, } @@ -88,21 +84,8 @@ impl DemoSource { "demo source color interval must be at least one frame" ); - let luma_len = (width * height) as usize; - let chroma_len = (width.div_ceil(2) * height.div_ceil(2)) as usize; - let planes = PALETTE - .iter() - .map(|&color| { - let (y, u, v) = yuv_from_rgb(color); - ( - Bytes::from(vec![y; luma_len]), - Bytes::from(vec![u; chroma_len]), - Bytes::from(vec![v; chroma_len]), - ) - }) - .collect(); - - Self { config, planes, started: None, frame_index: 0 } + let colors = PALETTE.iter().map(|&color| yuv_from_rgb(color)).collect(); + Self { config, colors, started: None, frame_index: 0 } } fn frame_interval(&self) -> Duration { @@ -121,7 +104,7 @@ impl PixelVideoSource for DemoSource { self.config.resolution } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self) -> Result, SourceError> { let started = *self.started.get_or_insert_with(Instant::now); // Pace against the ideal timeline so timestamps stay jitter-free. @@ -135,21 +118,21 @@ impl PixelVideoSource for DemoSource { let timestamp_us = elapsed.as_micros() as i64; let color_index = (elapsed.as_micros() / self.config.color_interval.as_micros().max(1)) as usize; - let (y, u, v) = self.planes[color_index % self.planes.len()].clone(); + let (y, u, v) = self.colors[color_index % self.colors.len()]; self.frame_index += 1; - let width = self.config.resolution.width; - Ok(Some(PixelVideoFrame { - resolution: self.config.resolution, + let VideoResolution { width, height } = self.config.resolution; + let mut buffer = I420Buffer::new(width, height); + let (data_y, data_u, data_v) = buffer.data_mut(); + data_y.fill(y); + data_u.fill(u); + data_v.fill(v); + + Ok(Some(VideoFrame { + rotation: VideoRotation::VideoRotation0, timestamp_us, - data: PixelVideoData::I420 { - y, - u, - v, - stride_y: width, - stride_u: width.div_ceil(2), - stride_v: width.div_ceil(2), - }, + frame_metadata: None, + buffer: Box::new(buffer), })) } } @@ -180,9 +163,10 @@ mod tests { let mut source = DemoSource::new(test_config()); let frame = source.next_frame().unwrap().unwrap(); - assert_eq!(frame.resolution, VideoResolution::new(64, 36)); + assert_eq!((frame.buffer.width(), frame.buffer.height()), (64, 36)); - let PixelVideoData::I420 { y, u, v, .. } = &frame.data; + let i420 = frame.buffer.as_i420().expect("demo source yields I420 buffers"); + let (y, u, v) = i420.data(); assert_eq!(y.len(), 64 * 36); assert_eq!(u.len(), 32 * 18); assert_eq!(v.len(), 32 * 18); @@ -202,10 +186,7 @@ mod tests { fn colors_cycle_at_the_color_interval() { let mut source = DemoSource::new(test_config()); - let luma = |frame: &PixelVideoFrame| { - let PixelVideoData::I420 { y, .. } = &frame.data; - y[0] - }; + let luma = |frame: &BoxVideoFrame| frame.buffer.as_i420().unwrap().data().0[0]; // Two frames per color at 1000 fps with a 2 ms interval. let first = source.next_frame().unwrap().unwrap(); diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 1c7296327..b0b36006a 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -23,11 +23,12 @@ use crate::{ encoded::{ h26x::{access_unit_from_annex_b, access_unit_from_h264_avc}, CodecSpecific, EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, - OwnedEncodedAccessUnit, RateControl, + OwnedEncodedAccessUnit, }, error::{CaptureError, SourceError}, primitive::VideoResolution, }; +use livekit::webrtc::video_source::EncodedRateControl; /// Encoded sample format expected from a GStreamer appsink. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -128,7 +129,7 @@ impl GStreamerEncoderRateControl { } } - fn update(&mut self, rate_control: RateControl) { + fn update(&mut self, rate_control: EncodedRateControl) { if self.last_target_bitrate_bps == Some(rate_control.target_bitrate_bps) { return; } @@ -258,7 +259,7 @@ impl EncodedVideoSource for GStreamerVideoSource { let _ = self.appsink.send_event(gst::event::CustomUpstream::new(structure)); } - fn update_rate_control(&mut self, rate_control: RateControl) { + fn update_rate_control(&mut self, rate_control: EncodedRateControl) { if let Some(control) = &mut self.rate_control { control.update(rate_control); } From 5e47723fde6273fe6684ee629cb233e799716036 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:04:49 -0700 Subject: [PATCH 26/56] Remove redundant constructor --- livekit-capture/src/sources/gstreamer.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index b0b36006a..eaa38d41c 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -75,18 +75,6 @@ pub struct GStreamerVideoSourceConfig { pub resolution: VideoResolution, } -impl GStreamerVideoSourceConfig { - /// Creates GStreamer appsink source configuration. - pub fn new( - sample_format: GStreamerSampleFormat, - start_timestamp_us: i64, - frame_interval_us: i64, - resolution: VideoResolution, - ) -> Self { - Self { sample_format, start_timestamp_us, frame_interval_us, resolution } - } -} - /// Bitrate unit used by a GStreamer encoder property. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GStreamerBitrateUnit { From 464ae220c0c78f7145312ece9e4bcd1367de160b Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:18:00 -0700 Subject: [PATCH 27/56] Apply segfault fix --- webrtc-sys/src/video_encoder_factory.cpp | 84 ++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/webrtc-sys/src/video_encoder_factory.cpp b/webrtc-sys/src/video_encoder_factory.cpp index d8b7fd454..e18449c87 100644 --- a/webrtc-sys/src/video_encoder_factory.cpp +++ b/webrtc-sys/src/video_encoder_factory.cpp @@ -17,16 +17,21 @@ #include "livekit/video_encoder_factory.h" #include +#include #include #include #include #include #include +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" + #include "api/environment/environment_factory.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory_template.h" +#include "livekit/encoded_video_frame_buffer.h" #include "livekit/objc_video_factory.h" #include "livekit/passthrough_video_encoder.h" #include "livekit/webrtc.h" @@ -605,6 +610,80 @@ VideoEncoderFactory::CodecSupport VideoEncoderFactory::QueryCodecSupport( return internal_factory_->QueryCodecSupport(format, scalability_mode); } +namespace { + +// Real encoders can never consume pre-encoded access units, but frames +// carrying an EncodedVideoFrameBuffer can still reach one in the window +// between stream startup and the sender's encoder selector switching onto +// the pass-through backend. Some platform encoders blind-cast native +// buffers (macOS ObjCVideoEncoder casts to ObjCFrameBuffer and retains a +// garbage pointer), so forwarding such a frame is a crash, not a graceful +// failure. Drop it instead: the selector switches shortly after, and the +// pass-through encoder requests a fresh keyframe when it takes over. +class EncodedFrameGuardEncoder final : public webrtc::VideoEncoder { + public: + explicit EncodedFrameGuardEncoder( + std::unique_ptr encoder) + : encoder_(std::move(encoder)) {} + + void SetFecControllerOverride( + webrtc::FecControllerOverride* fec_controller_override) override { + encoder_->SetFecControllerOverride(fec_controller_override); + } + + int InitEncode(const webrtc::VideoCodec* codec_settings, + const Settings& settings) override { + return encoder_->InitEncode(codec_settings, settings); + } + + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) override { + return encoder_->RegisterEncodeCompleteCallback(callback); + } + + int32_t Release() override { return encoder_->Release(); } + + int32_t Encode( + const webrtc::VideoFrame& frame, + const std::vector* frame_types) override { + if (livekit::EncodedVideoFrameBuffer::FromNative( + frame.video_frame_buffer().get())) { + static std::atomic logged{false}; + if (!logged.exchange(true)) { + RTC_LOG(LS_WARNING) + << "Dropping pre-encoded access unit sent to a non pass-through " + "encoder; waiting for the sender to switch onto the " + "pass-through backend"; + } + return WEBRTC_VIDEO_CODEC_OK; + } + return encoder_->Encode(frame, frame_types); + } + + void SetRates(const RateControlParameters& parameters) override { + encoder_->SetRates(parameters); + } + + void OnPacketLossRateUpdate(float packet_loss_rate) override { + encoder_->OnPacketLossRateUpdate(packet_loss_rate); + } + + void OnRttUpdate(int64_t rtt_ms) override { encoder_->OnRttUpdate(rtt_ms); } + + void OnLossNotification(const LossNotification& loss_notification) override { + encoder_->OnLossNotification(loss_notification); + } + + EncoderInfo GetEncoderInfo() const override { + return encoder_->GetEncoderInfo(); + } + + private: + std::unique_ptr encoder_; +}; + +} // namespace + std::unique_ptr VideoEncoderFactory::Create( const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) { @@ -614,6 +693,11 @@ std::unique_ptr VideoEncoderFactory::Create( env, internal_factory_.get(), nullptr, format); } + if (encoder && + BackendFromFormat(format) != VideoEncoderBackend::PreEncoded) { + encoder = std::make_unique(std::move(encoder)); + } + return encoder; } From e455b54d5c151e92bcf55ed76354cdd15e160bcb Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:18:14 -0700 Subject: [PATCH 28/56] Gstreamer source owns pipeline --- livekit-capture/README.md | 27 +-- livekit-capture/src/encoded/mod.rs | 18 +- livekit-capture/src/encoded/pump.rs | 15 +- livekit-capture/src/pixel/mod.rs | 13 +- livekit-capture/src/pixel/pump.rs | 18 +- livekit-capture/src/sources/demo.rs | 18 +- livekit-capture/src/sources/gstreamer.rs | 231 +++++++++++++++++------ 7 files changed, 251 insertions(+), 89 deletions(-) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index a1c9b93a6..4b45b0c6f 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -24,17 +24,22 @@ ingest source; the `demo` feature adds a synthetic pixel source for testing. spawn into the same `pump::RunningPump`, so an application supervises running pumps of either kind uniformly (`stop()`, `join_async()`, stats); the `pump` module holds this shared machinery. -- `sources::gstreamer::ensure_encoded_appsink` and friends turn an arbitrary - pipeline (containing `appsink name=lk_appsink` or one unlinked encoded pad) - into an encoded source; `encoded_caps_string` is the single per-codec caps - table. The GStreamer source answers keyframe requests with a - `GstForceKeyUnit` upstream event. +- `sources::gstreamer::GStreamerVideoSource` — built solely from + configuration (`GStreamerVideoSourceConfig`: launch description, codec, + resolution, optional rate-control binding). The source owns its pipeline: + it is started at construction, construction fails loudly on pipeline + problems, bus errors surface as source errors, and the pipeline stops when + the source is dropped. `encoded_caps_string` remains the single per-codec + caps table for writing producer pipelines. ## GStreamer ingest -`GStreamerVideoSource` implements `EncodedVideoSource` on top of an -`appsink` producing H.264 (Annex-B or AVC), H.265 Annex-B, VP8, VP9, or AV1 -access units. Drive it with an `EncodedVideoPump`, which builds the encoded -RTC source, derives the passthrough publish options, and forwards keyframe -and rate-control requests back to the pipeline. Passthrough is single-layer -(`L1T1`); access units carrying other layering metadata are rejected. +`GStreamerVideoSource` implements `EncodedVideoSource` on top of a pipeline +whose `appsink` (named `lk_appsink`, or attached automatically to one +unlinked encoded pad) produces H.264 (Annex-B or AVC), H.265 Annex-B, VP8, +VP9, or AV1 access units. Drive it with an `EncodedVideoPump`, which builds +the encoded RTC source, derives the passthrough publish options, and +forwards keyframe requests (answered with a `GstForceKeyUnit` upstream +event) and rate-control targets back to the pipeline. Passthrough is +single-layer (`L1T1`); access units carrying other layering metadata are +rejected. diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index 84de9bcfd..2f3855fc1 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -24,6 +24,7 @@ use crate::{ error::{CaptureError, SourceError}, primitive::VideoResolution, + pump::PumpStop, }; use bytes::Bytes; use livekit::{ @@ -52,7 +53,15 @@ pub trait EncodedVideoSource: Send { /// Blocks until the next access unit is available, returning `Ok(None)` /// when the source reaches the end of its stream. - fn next_access_unit(&mut self) -> Result, SourceError>; + /// + /// Sources must return promptly (with `Ok(None)`) once `stop` fires: + /// integrate it into the blocking wait, or bound each wait so the token + /// is observed within a frame interval or so. The pump distinguishes a + /// stop from end of stream via the token. + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError>; /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to /// the producer so it can emit an IDR. @@ -462,8 +471,11 @@ impl EncodedVideoSource for Box { (**self).codec() } - fn next_access_unit(&mut self) -> Result, SourceError> { - (**self).next_access_unit() + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError> { + (**self).next_access_unit(stop) } fn request_keyframe(&mut self) { diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs index 19228df72..b6af2f204 100644 --- a/livekit-capture/src/encoded/pump.rs +++ b/livekit-capture/src/encoded/pump.rs @@ -115,8 +115,14 @@ impl EncodedVideoPump { self.source.request_keyframe(); } - let Some(access_unit) = self.source.next_access_unit()? else { - break PumpExit::EndOfStream; + let Some(access_unit) = self.source.next_access_unit(&self.stop)? else { + // `None` is end of stream, unless the source returned early + // because the stop handle fired mid-wait. + break if self.stop.is_stopped() { + PumpExit::Stopped + } else { + PumpExit::EndOfStream + }; }; // Drop pre-roll deltas: decoding can only start at a keyframe. @@ -224,7 +230,10 @@ mod tests { EncodedVideoCodec::VP8 } - fn next_access_unit(&mut self) -> Result, SourceError> { + fn next_access_unit( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { Ok(self.access_units.pop_front()) } } diff --git a/livekit-capture/src/pixel/mod.rs b/livekit-capture/src/pixel/mod.rs index e64e1fdde..cb7f1cf82 100644 --- a/livekit-capture/src/pixel/mod.rs +++ b/livekit-capture/src/pixel/mod.rs @@ -28,7 +28,7 @@ use livekit::webrtc::video_frame::BoxVideoFrame; pub use pump::PixelVideoPump; -use crate::{error::SourceError, primitive::VideoResolution}; +use crate::{error::SourceError, primitive::VideoResolution, pump::PumpStop}; /// Source of pixel (unencoded) video frames, such as a camera device. pub trait PixelVideoSource: Send { @@ -38,9 +38,14 @@ pub trait PixelVideoSource: Send { /// Blocks until the next frame is available, returning `Ok(None)` when /// the source reaches the end of its stream. /// + /// Sources must return promptly (with `Ok(None)`) once `stop` fires: + /// integrate it into the blocking wait, or bound each wait so the token + /// is observed within a frame interval or so. The pump distinguishes a + /// stop from end of stream via the token. + /// /// Sources may pre-fill the frame's `frame_metadata`; a metadata /// callback set on the pump takes precedence when it returns `Some`. - fn next_frame(&mut self) -> Result, SourceError>; + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError>; } impl PixelVideoSource for Box { @@ -48,8 +53,8 @@ impl PixelVideoSource for Box { (**self).resolution() } - fn next_frame(&mut self) -> Result, SourceError> { - (**self).next_frame() + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + (**self).next_frame(stop) } } diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs index d618a47ec..7bb98329f 100644 --- a/livekit-capture/src/pixel/pump.rs +++ b/livekit-capture/src/pixel/pump.rs @@ -97,8 +97,14 @@ impl PixelVideoPump { if self.stop.is_stopped() { break PumpExit::Stopped; } - let Some(mut frame) = self.source.next_frame()? else { - break PumpExit::EndOfStream; + let Some(mut frame) = self.source.next_frame(&self.stop)? else { + // `None` is end of stream, unless the source returned early + // because the stop handle fired mid-wait. + break if self.stop.is_stopped() { + PumpExit::Stopped + } else { + PumpExit::EndOfStream + }; }; if let Some(metadata) = self.frame_metadata.as_mut().and_then(|callback| callback(&frame)) @@ -177,7 +183,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { Ok(self.frames.pop_front()) } } @@ -234,7 +240,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { panic!("source exploded"); } } @@ -258,7 +264,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { std::thread::sleep(std::time::Duration::from_millis(1)); Ok(Some(pixel_frame(0))) } @@ -283,7 +289,7 @@ mod tests { RESOLUTION } - fn next_frame(&mut self) -> Result, SourceError> { + fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { std::thread::sleep(std::time::Duration::from_millis(1)); Ok(Some(pixel_frame(0))) } diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index 07fffe296..5dad9b8d1 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -14,7 +14,7 @@ //! Solid-color demo source for testing. -use crate::{error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution}; +use crate::{error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop}; use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; use std::{ thread, @@ -104,7 +104,9 @@ impl PixelVideoSource for DemoSource { self.config.resolution } - fn next_frame(&mut self) -> Result, SourceError> { + // Sleeps at most one frame interval, so the stop token is observed + // promptly without integrating it into the wait. + fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { let started = *self.started.get_or_insert_with(Instant::now); // Pace against the ideal timeline so timestamps stay jitter-free. @@ -162,7 +164,7 @@ mod tests { fn yields_frames_with_configured_dimensions() { let mut source = DemoSource::new(test_config()); - let frame = source.next_frame().unwrap().unwrap(); + let frame = source.next_frame(&PumpStop::new()).unwrap().unwrap(); assert_eq!((frame.buffer.width(), frame.buffer.height()), (64, 36)); let i420 = frame.buffer.as_i420().expect("demo source yields I420 buffers"); @@ -176,8 +178,8 @@ mod tests { fn timestamps_follow_the_frame_rate() { let mut source = DemoSource::new(test_config()); - let first = source.next_frame().unwrap().unwrap(); - let second = source.next_frame().unwrap().unwrap(); + let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); + let second = source.next_frame(&PumpStop::new()).unwrap().unwrap(); assert_eq!(first.timestamp_us, 0); assert_eq!(second.timestamp_us, 1_000); } @@ -189,9 +191,9 @@ mod tests { let luma = |frame: &BoxVideoFrame| frame.buffer.as_i420().unwrap().data().0[0]; // Two frames per color at 1000 fps with a 2 ms interval. - let first = source.next_frame().unwrap().unwrap(); - let same_color = source.next_frame().unwrap().unwrap(); - let next_color = source.next_frame().unwrap().unwrap(); + let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); + let same_color = source.next_frame(&PumpStop::new()).unwrap().unwrap(); + let next_color = source.next_frame(&PumpStop::new()).unwrap().unwrap(); assert_eq!(luma(&first), luma(&same_color)); assert_ne!(luma(&first), luma(&next_color)); } diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index eaa38d41c..44ee69f0a 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -27,6 +27,7 @@ use crate::{ }, error::{CaptureError, SourceError}, primitive::VideoResolution, + pump::PumpStop, }; use livekit::webrtc::video_source::EncodedRateControl; @@ -62,17 +63,43 @@ impl GStreamerSampleFormat { } } -/// Configuration for a GStreamer appsink encoded source. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Configuration for a GStreamer encoded video source. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct GStreamerVideoSourceConfig { - /// Format of encoded buffers pulled from appsink. - pub sample_format: GStreamerSampleFormat, - /// Timestamp added to the first buffer timestamp, or used directly as fallback. - pub start_timestamp_us: i64, - /// Fallback frame interval when a GStreamer buffer has no PTS or DTS. - pub frame_interval_us: i64, - /// Encoded frame resolution in pixels. + /// GStreamer launch description for the encoded producer pipeline. + /// + /// Must contain `appsink name=lk_appsink`, or leave exactly one encoded + /// video source pad unlinked for the source to attach one to. + pub pipeline: String, + + /// Codec expected from the pipeline; inferred from pipeline caps when + /// `None`. + pub codec: Option, + + /// Encoded frame resolution. pub resolution: VideoResolution, + + /// Nominal frame rate, used for fallback frame timing when pipeline + /// buffers carry no timestamps. + pub framerate_fps: u32, + + /// Forwards WebRTC rate-control targets to an encoder element's bitrate + /// property. Without this, the pipeline encodes at a fixed bitrate. + pub rate_control: Option, +} + +/// Binding from WebRTC rate-control targets to a GStreamer encoder property. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GStreamerRateControlConfig { + /// Name of the encoder element in the pipeline (e.g. `lk_encoder`). + pub element: String, + + /// Bitrate property to set on the element (e.g. `bitrate` for x264enc, + /// `target-bitrate` for vp8enc/vp9enc). + pub property: String, + + /// Unit the property expects. + pub unit: GStreamerBitrateUnit, } /// Bitrate unit used by a GStreamer encoder property. @@ -95,7 +122,7 @@ impl GStreamerBitrateUnit { /// GStreamer encoder bitrate control used by [`GStreamerVideoSource`]. #[derive(Debug, Clone)] -pub struct GStreamerEncoderRateControl { +struct GStreamerEncoderRateControl { encoder: gst::Element, bitrate_property: String, bitrate_unit: GStreamerBitrateUnit, @@ -104,7 +131,7 @@ pub struct GStreamerEncoderRateControl { impl GStreamerEncoderRateControl { /// Creates bitrate control for a GStreamer encoder element. - pub fn new( + fn new( encoder: gst::Element, bitrate_property: &str, bitrate_unit: GStreamerBitrateUnit, @@ -137,44 +164,106 @@ impl GStreamerEncoderRateControl { } } -/// Encoded source backed by a GStreamer appsink. +/// How long one appsink wait may block before the stop token is rechecked. +const SAMPLE_WAIT: gst::ClockTime = gst::ClockTime::from_mseconds(100); + +/// Encoded source that owns a GStreamer pipeline ending in an appsink. #[derive(Debug)] pub struct GStreamerVideoSource { + pipeline: gst::Pipeline, + bus: gst::Bus, appsink: gst_app::AppSink, - config: GStreamerVideoSourceConfig, + sample_format: GStreamerSampleFormat, + resolution: VideoResolution, + frame_interval_us: i64, next_fallback_timestamp_us: i64, rate_control: Option, } impl GStreamerVideoSource { - /// Creates an encoded source from an existing GStreamer appsink. - pub fn new(appsink: gst_app::AppSink, config: GStreamerVideoSourceConfig) -> Self { - Self { - appsink, - config, - next_fallback_timestamp_us: config.start_timestamp_us, - rate_control: None, + /// Builds, owns, and starts a GStreamer pipeline from configuration. + /// + /// The pipeline is set to `Playing` immediately — the appsink buffers a + /// bounded number of samples until a pump starts pulling — and returned + /// to `Null` when the source is dropped. Construction fails loudly on an + /// invalid launch description, a missing appsink or encoded pad, a + /// missing rate-control element, or a pipeline that refuses to start. + pub fn new(config: GStreamerVideoSourceConfig) -> Result { + if config.framerate_fps == 0 { + return Err(SourceError::new(GStreamerVideoSourceError::InvalidConfig( + "framerate_fps must be greater than zero", + ))); } - } - - /// Sets the encoder bitrate control used for downstream rate requests. - pub fn set_encoder_rate_control(&mut self, rate_control: GStreamerEncoderRateControl) { - self.rate_control = Some(rate_control); - } + gst::init().map_err(|err| { + SourceError::new(GStreamerVideoSourceError::Pipeline(format!( + "failed to initialize GStreamer: {err}" + ))) + })?; - /// Returns the wrapped appsink. - pub fn appsink(&self) -> &gst_app::AppSink { - &self.appsink - } + let pipeline = gst::parse::launch(&config.pipeline) + .map_err(|err| { + SourceError::new(GStreamerVideoSourceError::Pipeline(format!( + "failed to create pipeline: {err}" + ))) + })? + .downcast::() + .map_err(|_| SourceError::new(GStreamerVideoSourceError::NotAPipeline))?; + + let (appsink, sample_format) = ensure_encoded_appsink(&pipeline, config.codec) + .map_err(|err| SourceError::new(GStreamerVideoSourceError::Layout(err)))?; + + let rate_control = config + .rate_control + .map(|binding| -> Result { + let encoder = pipeline.by_name(&binding.element).ok_or_else(|| { + GStreamerVideoSourceError::MissingRateControlElement(binding.element.clone()) + })?; + Ok(GStreamerEncoderRateControl::new(encoder, &binding.property, binding.unit)) + }) + .transpose() + .map_err(SourceError::new)?; + + let bus = pipeline.bus().ok_or_else(|| { + SourceError::new(GStreamerVideoSourceError::Pipeline( + "pipeline has no message bus".to_owned(), + )) + })?; - /// Returns the source configuration. - pub fn config(&self) -> GStreamerVideoSourceConfig { - self.config - } + pipeline.set_state(gst::State::Playing).map_err(|err| { + SourceError::new(GStreamerVideoSourceError::Pipeline(format!( + "failed to start pipeline: {err}" + ))) + })?; - /// Consumes this source and returns the wrapped appsink. - pub fn into_appsink(self) -> gst_app::AppSink { - self.appsink + Ok(Self { + pipeline, + bus, + appsink, + sample_format, + resolution: config.resolution, + frame_interval_us: 1_000_000 / i64::from(config.framerate_fps), + next_fallback_timestamp_us: 0, + rate_control, + }) + } + + /// Returns the owned pipeline. + pub fn pipeline(&self) -> &gst::Pipeline { + &self.pipeline + } + + /// Surfaces a pipeline bus error, if one is pending. + fn check_bus(&self) -> Result<(), GStreamerVideoSourceError> { + while let Some(message) = self.bus.pop_filtered(&[gst::MessageType::Error]) { + if let gst::MessageView::Error(error) = message.view() { + return Err(GStreamerVideoSourceError::Pipeline(format!( + "{} ({})", + error.error(), + error.debug().map(|s| s.to_string()).unwrap_or_default(), + ))); + } + } + Ok(()) } fn access_unit_from_sample( @@ -194,46 +283,68 @@ impl GStreamerVideoSource { .map_err(|err| GStreamerVideoSourceError::MapReadable(err.to_string()))?; let payload = map.as_ref(); access_unit_from_sample_payload( - self.config.sample_format, + self.sample_format, payload, timestamp_us, frame_type, - self.config.resolution, + self.resolution, ) .map_err(GStreamerVideoSourceError::Capture) } fn timestamp_us(&mut self, buffer: &gst::BufferRef) -> i64 { if let Some(timestamp) = buffer.pts().or_else(|| buffer.dts()) { - let timestamp_us = - clock_time_to_timestamp_us(self.config.start_timestamp_us, timestamp); + let timestamp_us = clock_time_to_timestamp_us(0, timestamp); self.next_fallback_timestamp_us = - timestamp_us.saturating_add(self.config.frame_interval_us); + timestamp_us.saturating_add(self.frame_interval_us); return timestamp_us; } let timestamp_us = self.next_fallback_timestamp_us; self.next_fallback_timestamp_us = - self.next_fallback_timestamp_us.saturating_add(self.config.frame_interval_us); + self.next_fallback_timestamp_us.saturating_add(self.frame_interval_us); timestamp_us } } +impl Drop for GStreamerVideoSource { + fn drop(&mut self) { + // Returning the pipeline to `Null` releases its resources; GStreamer + // does not stop a running pipeline on the last unref. + let _ = self.pipeline.set_state(gst::State::Null); + } +} + impl EncodedVideoSource for GStreamerVideoSource { fn resolution(&self) -> VideoResolution { - self.config.resolution + self.resolution } fn codec(&self) -> EncodedVideoCodec { - self.config.sample_format.codec() + self.sample_format.codec() } - fn next_access_unit(&mut self) -> Result, SourceError> { - match self.appsink.pull_sample() { - Ok(sample) => self.access_unit_from_sample(&sample).map(Some).map_err(SourceError::new), - Err(_err) if self.appsink.is_eos() => Ok(None), - Err(err) => { - Err(SourceError::new(GStreamerVideoSourceError::PullSample(err.to_string()))) + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError> { + // Bounded waits keep the stop token observed within `SAMPLE_WAIT` + // even while the pipeline produces nothing. + loop { + if stop.is_stopped() { + return Ok(None); + } + self.check_bus().map_err(SourceError::new)?; + + match self.appsink.try_pull_sample(SAMPLE_WAIT) { + Some(sample) => { + return self + .access_unit_from_sample(&sample) + .map(Some) + .map_err(SourceError::new); + } + None if self.appsink.is_eos() => return Ok(None), + None => {} } } } @@ -307,9 +418,21 @@ fn clamp_to_i64(value: u64, minimum: i64, maximum: i64) -> i64 { /// Error returned by GStreamer appsink encoded sources. #[derive(Debug, Error)] pub enum GStreamerVideoSourceError { - /// The appsink failed to produce a sample. - #[error("failed to pull GStreamer appsink sample: {0}")] - PullSample(String), + /// Configuration is invalid. + #[error("invalid GStreamer source configuration: {0}")] + InvalidConfig(&'static str), + /// The launch description did not produce a pipeline. + #[error("GStreamer description did not create a pipeline")] + NotAPipeline, + /// The rate-control element is missing from the pipeline. + #[error("pipeline has no element named '{0}' for rate control")] + MissingRateControlElement(String), + /// The pipeline could not be built or started, or errored at runtime. + #[error("GStreamer pipeline error: {0}")] + Pipeline(String), + /// The pipeline layout cannot feed an encoded appsink. + #[error(transparent)] + Layout(#[from] GStreamerPipelineError), /// The sample did not contain an encoded buffer. #[error("GStreamer sample did not contain a buffer")] MissingBuffer, From 360666190b13e7c0b476e98d1d8ed0b4760393aa Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:06:49 -0700 Subject: [PATCH 29/56] Automatically detect resolution for gstreamer --- livekit-capture/Cargo.toml | 5 + livekit-capture/README.md | 18 +- livekit-capture/src/primitive.rs | 12 ++ livekit-capture/src/sources/gstreamer.rs | 216 ++++++++++++++++++++--- 4 files changed, 223 insertions(+), 28 deletions(-) diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 810a0dfe6..0e79bc916 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -22,6 +22,11 @@ tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] default = ["demo", "gstreamer"] # TODO: Remove after testing +# Async source constructors (`new`), which run blocking construction and +# stream discovery on the tokio blocking pool. Without this feature only the +# `new_blocking` constructors are available. +tokio = ["tokio/rt"] + # Pixel sources demo = [] diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 4b45b0c6f..da90af278 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -25,12 +25,18 @@ ingest source; the `demo` feature adds a synthetic pixel source for testing. running pumps of either kind uniformly (`stop()`, `join_async()`, stats); the `pump` module holds this shared machinery. - `sources::gstreamer::GStreamerVideoSource` — built solely from - configuration (`GStreamerVideoSourceConfig`: launch description, codec, - resolution, optional rate-control binding). The source owns its pipeline: - it is started at construction, construction fails loudly on pipeline - problems, bus errors surface as source errors, and the pipeline stops when - the source is dropped. `encoded_caps_string` remains the single per-codec - caps table for writing producer pipelines. + configuration (`GStreamerVideoSourceConfig`: launch description, plus + optional codec, resolution, and rate-control binding). The source owns its + pipeline: it is started at construction, construction fails loudly on + pipeline problems, bus errors surface as source errors, and the pipeline + stops when the source is dropped. Codec and resolution are discovered from + pipeline caps when omitted (a declared resolution skips the discovery wait + and is verified against the stream); a mid-stream caps change is an error + until track republication is supported. With the `tokio` crate feature, + `new` runs construction and discovery on the blocking pool — the + convention for all backends — while `new_blocking` serves non-async + consumers. `encoded_caps_string` remains the single per-codec caps table + for writing producer pipelines. ## GStreamer ingest diff --git a/livekit-capture/src/primitive.rs b/livekit-capture/src/primitive.rs index 0a0e1b05f..0fb599d65 100644 --- a/livekit-capture/src/primitive.rs +++ b/livekit-capture/src/primitive.rs @@ -62,6 +62,18 @@ impl VideoResolution { } } +impl std::fmt::Display for VideoResolution { + /// Formats as `WIDTHxHEIGHT`. + /// + /// ``` + /// # use livekit_capture::primitive::VideoResolution; + /// assert_eq!(VideoResolution::new(1920, 1080).to_string(), "1920x1080"); + /// ``` + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}x{}", self.width, self.height) + } +} + impl From for livekit::webrtc::video_source::VideoResolution { fn from(value: VideoResolution) -> Self { Self { width: value.width, height: value.height } diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 44ee69f0a..86074b826 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -77,11 +77,12 @@ pub struct GStreamerVideoSourceConfig { pub codec: Option, /// Encoded frame resolution. - pub resolution: VideoResolution, - - /// Nominal frame rate, used for fallback frame timing when pipeline - /// buffers carry no timestamps. - pub framerate_fps: u32, + /// + /// When `None`, the resolution is discovered from the first sample's + /// negotiated caps — construction then waits for the pipeline to produce + /// data. When set, construction returns without waiting, and the first + /// sample is verified against the declared resolution. + pub resolution: Option, /// Forwards WebRTC rate-control targets to an encoder element's bitrate /// property. Without this, the pipeline encodes at a fixed bitrate. @@ -167,6 +168,12 @@ impl GStreamerEncoderRateControl { /// How long one appsink wait may block before the stop token is rechecked. const SAMPLE_WAIT: gst::ClockTime = gst::ClockTime::from_mseconds(100); +/// How long stream discovery waits for the pipeline's first sample. +const DISCOVERY_TIMEOUT: gst::ClockTime = gst::ClockTime::from_seconds(5); + +/// Fallback frame interval when neither caps nor buffers carry timing. +const DEFAULT_FRAME_INTERVAL_US: i64 = 1_000_000 / 30; + /// Encoded source that owns a GStreamer pipeline ending in an appsink. #[derive(Debug)] pub struct GStreamerVideoSource { @@ -178,9 +185,29 @@ pub struct GStreamerVideoSource { frame_interval_us: i64, next_fallback_timestamp_us: i64, rate_control: Option, + /// Caps the stream has been validated against; a pointer change on a + /// later sample triggers revalidation. + negotiated_caps: Option, + /// Sample pulled during stream discovery, handed out first. + pending_sample: Option, } impl GStreamerVideoSource { + /// Creates the source, running blocking construction and stream + /// discovery on the tokio blocking pool. + /// + /// Requires a running tokio runtime. This is the async-constructor + /// convention for capture backends: `new` for async consumers, and + /// [`GStreamerVideoSource::new_blocking`] for everything else. + #[cfg(feature = "tokio")] + pub async fn new(config: GStreamerVideoSourceConfig) -> Result { + match tokio::task::spawn_blocking(move || Self::new_blocking(config)).await { + Ok(result) => result, + Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), + Err(err) => Err(SourceError::new(err)), + } + } + /// Builds, owns, and starts a GStreamer pipeline from configuration. /// /// The pipeline is set to `Playing` immediately — the appsink buffers a @@ -188,12 +215,11 @@ impl GStreamerVideoSource { /// to `Null` when the source is dropped. Construction fails loudly on an /// invalid launch description, a missing appsink or encoded pad, a /// missing rate-control element, or a pipeline that refuses to start. - pub fn new(config: GStreamerVideoSourceConfig) -> Result { - if config.framerate_fps == 0 { - return Err(SourceError::new(GStreamerVideoSourceError::InvalidConfig( - "framerate_fps must be greater than zero", - ))); - } + /// + /// When the configuration declares no resolution, this blocks until the + /// first sample arrives (bounded by a discovery timeout) to read the + /// negotiated stream settings. + pub fn new_blocking(config: GStreamerVideoSourceConfig) -> Result { gst::init().map_err(|err| { SourceError::new(GStreamerVideoSourceError::Pipeline(format!( "failed to initialize GStreamer: {err}" @@ -235,16 +261,56 @@ impl GStreamerVideoSource { ))) })?; - Ok(Self { + let mut source = Self { pipeline, bus, appsink, sample_format, - resolution: config.resolution, - frame_interval_us: 1_000_000 / i64::from(config.framerate_fps), + resolution: config.resolution.unwrap_or_default(), + frame_interval_us: DEFAULT_FRAME_INTERVAL_US, next_fallback_timestamp_us: 0, rate_control, - }) + negotiated_caps: None, + pending_sample: None, + }; + + // Without a declared resolution, discover the stream settings from + // the first sample's negotiated caps; the sample is buffered so no + // keyframe is lost. A declared resolution skips the wait and is + // verified lazily against the first sample instead. + if config.resolution.is_none() { + let sample = source.wait_first_sample().map_err(SourceError::new)?; + let caps = + sample.caps().ok_or(GStreamerVideoSourceError::MissingResolutionCaps).map_err(SourceError::new)?; + source.resolution = resolution_from_caps(caps) + .ok_or(GStreamerVideoSourceError::MissingResolutionCaps) + .map_err(SourceError::new)?; + if let Some(frame_interval_us) = frame_interval_from_caps(caps) { + source.frame_interval_us = frame_interval_us; + } + source.negotiated_caps = Some(caps.to_owned()); + source.pending_sample = Some(sample); + } + + Ok(source) + } + + /// Blocks until the pipeline produces its first sample, surfacing bus + /// errors and bounding the wait by the discovery timeout. + fn wait_first_sample(&self) -> Result { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(DISCOVERY_TIMEOUT.seconds()); + loop { + self.check_bus()?; + if let Some(sample) = self.appsink.try_pull_sample(SAMPLE_WAIT) { + return Ok(sample); + } + if self.appsink.is_eos() { + return Err(GStreamerVideoSourceError::EndedBeforeFirstSample); + } + if std::time::Instant::now() >= deadline { + return Err(GStreamerVideoSourceError::DiscoveryTimeout); + } + } } /// Returns the owned pipeline. @@ -266,6 +332,64 @@ impl GStreamerVideoSource { Ok(()) } + /// Validates a sample's caps against the established stream settings. + /// + /// Caps are immutable and refcounted, so an unchanged stream passes with + /// a pointer comparison. On a caps change, the declared or discovered + /// resolution and codec must match: live stream reconfiguration would + /// require republishing the track, which is not supported yet. + fn check_caps(&mut self, sample: &gst::Sample) -> Result<(), GStreamerVideoSourceError> { + let Some(caps) = sample.caps() else { + return Ok(()); + }; + if let Some(seen) = &self.negotiated_caps { + if seen.as_ptr() == caps.as_ptr() { + return Ok(()); + } + } + let had_baseline = self.negotiated_caps.is_some(); + + if let Some(structure) = caps.structure(0) { + if let Some(codec) = codec_from_caps_name(structure.name()) { + if codec != self.sample_format.codec() { + return Err(GStreamerVideoSourceError::Renegotiated { + from: format!("{:?}", self.sample_format.codec()), + to: format!("{codec:?}"), + }); + } + } + } + if let Some(resolution) = resolution_from_caps(caps) { + if resolution != self.resolution { + return Err(if had_baseline { + GStreamerVideoSourceError::Renegotiated { + from: self.resolution.to_string(), + to: resolution.to_string(), + } + } else { + GStreamerVideoSourceError::ResolutionMismatch { + configured: self.resolution, + actual: resolution, + } + }); + } + } + if let Some(frame_interval_us) = frame_interval_from_caps(caps) { + self.frame_interval_us = frame_interval_us; + } + + self.negotiated_caps = Some(caps.to_owned()); + Ok(()) + } + + fn process_sample( + &mut self, + sample: &gst::Sample, + ) -> Result { + self.check_caps(sample)?; + self.access_unit_from_sample(sample) + } + fn access_unit_from_sample( &mut self, sample: &gst::Sample, @@ -328,6 +452,10 @@ impl EncodedVideoSource for GStreamerVideoSource { &mut self, stop: &PumpStop, ) -> Result, SourceError> { + if let Some(sample) = self.pending_sample.take() { + return self.process_sample(&sample).map(Some).map_err(SourceError::new); + } + // Bounded waits keep the stop token observed within `SAMPLE_WAIT` // even while the pipeline produces nothing. loop { @@ -338,10 +466,7 @@ impl EncodedVideoSource for GStreamerVideoSource { match self.appsink.try_pull_sample(SAMPLE_WAIT) { Some(sample) => { - return self - .access_unit_from_sample(&sample) - .map(Some) - .map_err(SourceError::new); + return self.process_sample(&sample).map(Some).map_err(SourceError::new); } None if self.appsink.is_eos() => return Ok(None), None => {} @@ -418,12 +543,42 @@ fn clamp_to_i64(value: u64, minimum: i64, maximum: i64) -> i64 { /// Error returned by GStreamer appsink encoded sources. #[derive(Debug, Error)] pub enum GStreamerVideoSourceError { - /// Configuration is invalid. - #[error("invalid GStreamer source configuration: {0}")] - InvalidConfig(&'static str), /// The launch description did not produce a pipeline. #[error("GStreamer description did not create a pipeline")] NotAPipeline, + /// The pipeline produced no data during stream discovery. + #[error( + "pipeline produced no data during stream discovery; declare `resolution` in the \ + configuration to skip discovery, or check that the pipeline produces encoded video" + )] + DiscoveryTimeout, + /// The stream ended before producing a sample. + #[error("pipeline reached end of stream before producing a sample")] + EndedBeforeFirstSample, + /// Negotiated caps carry no resolution to discover. + #[error( + "negotiated caps declare no resolution; declare `resolution` in the configuration" + )] + MissingResolutionCaps, + /// The pipeline produces a different resolution than configured. + #[error("pipeline produces {actual}, but the configuration declares {configured}")] + ResolutionMismatch { + /// Resolution declared in the configuration. + configured: VideoResolution, + /// Resolution the pipeline negotiated. + actual: VideoResolution, + }, + /// Stream settings changed mid-stream. + #[error( + "pipeline renegotiated {from} to {to}; changing stream settings requires republishing \ + the track, which is not supported yet" + )] + Renegotiated { + /// Established stream setting. + from: String, + /// Newly negotiated stream setting. + to: String, + }, /// The rate-control element is missing from the pipeline. #[error("pipeline has no element named '{0}' for rate control")] MissingRateControlElement(String), @@ -485,6 +640,23 @@ fn access_unit_from_sample_payload( } } +/// Reads the frame resolution from negotiated caps, when declared. +fn resolution_from_caps(caps: &gst::CapsRef) -> Option { + let structure = caps.structure(0)?; + let width = structure.get::("width").ok()?; + let height = structure.get::("height").ok()?; + (width > 0 && height > 0) + .then(|| VideoResolution::new(width as u32, height as u32)) +} + +/// Derives the fallback frame interval from the caps framerate, when +/// declared and non-zero. +fn frame_interval_from_caps(caps: &gst::CapsRef) -> Option { + let framerate = caps.structure(0)?.get::("framerate").ok()?; + let (numer, denom) = (i64::from(framerate.numer()), i64::from(framerate.denom())); + (numer > 0 && denom > 0).then(|| 1_000_000 * denom / numer) +} + fn clock_time_to_timestamp_us(start_timestamp_us: i64, timestamp: gst::ClockTime) -> i64 { let timestamp_us = timestamp.useconds().min(i64::MAX as u64) as i64; start_timestamp_us.saturating_add(timestamp_us) From cb7efed2094aba0d90dfa67703f86dd3668b931d Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:38:58 -0700 Subject: [PATCH 30/56] Expose capture over FFI --- Cargo.lock | 1 + livekit-ffi/Cargo.toml | 10 + livekit-ffi/protocol/capture.proto | 160 ++++++++++++ livekit-ffi/protocol/ffi.proto | 21 +- livekit-ffi/protocol/room.proto | 2 + livekit-ffi/src/conversion/capture.rs | 89 +++++++ livekit-ffi/src/conversion/mod.rs | 2 + livekit-ffi/src/conversion/room.rs | 3 + livekit-ffi/src/server/capture.rs | 346 ++++++++++++++++++++++++++ livekit-ffi/src/server/mod.rs | 2 + livekit-ffi/src/server/requests.rs | 15 ++ 11 files changed, 648 insertions(+), 3 deletions(-) create mode 100644 livekit-ffi/protocol/capture.proto create mode 100644 livekit-ffi/src/conversion/capture.rs create mode 100644 livekit-ffi/src/server/capture.rs diff --git a/Cargo.lock b/Cargo.lock index fbd0a6461..b11f4087b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4241,6 +4241,7 @@ dependencies = [ "link-cplusplus", "livekit", "livekit-api", + "livekit-capture", "livekit-protocol", "log", "parking_lot", diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index daed64c91..5c7252856 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -19,8 +19,18 @@ __rustls-tls = ["livekit/__rustls-tls"] # Enable tokio-console to debug tasks tracing = ["tokio/tracing", "console-subscriber"] +# Capture sources (livekit-capture): publish tracks from server-side +# producers such as GStreamer pipelines. Links system GStreamer. +capture = [ + "dep:livekit-capture", + "livekit-capture/demo", + "livekit-capture/gstreamer", + "livekit-capture/tokio", +] + [dependencies] livekit = { workspace = true } +livekit-capture = { workspace = true, optional = true, default-features = false } webrtc-sys = { workspace = true } soxr-sys = { workspace = true } imgproc = { workspace = true } diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto new file mode 100644 index 000000000..3c391e08f --- /dev/null +++ b/livekit-ffi/protocol/capture.proto @@ -0,0 +1,160 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package livekit.proto; +option csharp_namespace = "LiveKit.Proto"; + +import "handle.proto"; +import "room.proto"; +import "video_frame.proto"; + +// Capture sources (livekit-capture) +// +// A capture source owns a media producer (e.g. a GStreamer pipeline) and the +// pump that forwards its frames into an RTC video source, so no per-frame +// FFI traffic is needed. These requests are only served when the FFI server +// is built with the `capture` feature; otherwise they fail with an error. +// +// Usage: +// 1. NewCaptureSourceRequest (async): builds and starts the producer, +// discovers stream settings, and returns an OwnedCaptureSource. Its info +// embeds an OwnedVideoSource for the existing CreateVideoTrackRequest and +// the recommended TrackPublishOptions for PublishTrackRequest. +// 2. StartCaptureRequest: starts pumping frames into the RTC source. +// 3. The capture runs until StopCaptureRequest, end of stream, or an error; +// a CaptureSourceEvent is delivered exactly once when it ends. + +// Bitrate unit expected by a GStreamer encoder property. +enum GstreamerBitrateUnit { + GSTREAMER_BITRATE_UNIT_BPS = 0; + GSTREAMER_BITRATE_UNIT_KBPS = 1; +} + +// Binding from WebRTC rate-control targets to a GStreamer encoder property. +message GstreamerRateControl { + // Name of the encoder element in the pipeline (e.g. `lk_encoder`). + required string element = 1; + // Bitrate property to set on the element (e.g. `bitrate` for x264enc, + // `target-bitrate` for vp8enc/vp9enc). + required string property = 2; + // Unit the property expects. + required GstreamerBitrateUnit unit = 3; +} + +// Encoded ingest from a GStreamer pipeline. +message GstreamerVideoSourceConfig { + // GStreamer launch description for the encoded producer pipeline. + // Must contain `appsink name=lk_appsink`, or leave exactly one encoded + // video source pad unlinked for the source to attach one to. + required string pipeline = 1; + // Codec expected from the pipeline; inferred from pipeline caps when + // omitted. + optional VideoCodec codec = 2; + // Encoded frame resolution. When omitted, it is discovered from the + // pipeline's negotiated caps; when set, the pipeline output is verified + // against it. + optional VideoSourceResolution resolution = 3; + // Forwards WebRTC rate-control targets to an encoder element's bitrate + // property. Without this, the pipeline encodes at a fixed bitrate. + optional GstreamerRateControl rate_control = 4; +} + +// Test source producing solid-color frames, cycling through a palette. +message DemoVideoSourceConfig {} + +// Kind of media a capture source produces. +enum CaptureSourceKind { + // Pixel frames, published through the WebRTC encoder. + CAPTURE_SOURCE_PIXEL = 0; + // Pre-encoded access units, published as passthrough. + CAPTURE_SOURCE_ENCODED = 1; +} + +message CaptureSourceInfo { + required CaptureSourceKind kind = 1; + // Declared or discovered stream resolution. + required VideoSourceResolution resolution = 2; + // Codec produced by the source; encoded sources only. + optional VideoCodec codec = 3; + // Publish options derived from the source (codec, encoder backend, ...). + // Merge application options (e.g. track source) over these when calling + // PublishTrackRequest. + required TrackPublishOptions recommended_publish_options = 4; + // RTC video source fed by this capture source; pass its handle to + // CreateVideoTrackRequest. Owned by the client like any other handle. + required OwnedVideoSource video_source = 5; +} + +message OwnedCaptureSource { + required FfiOwnedHandle handle = 1; + required CaptureSourceInfo info = 2; +} + +// Create a new capture source from configuration. +// +// Completes asynchronously with a NewCaptureSourceCallback: construction +// starts the producer and may wait for its first output to discover stream +// settings. +message NewCaptureSourceRequest { + oneof config { + GstreamerVideoSourceConfig gstreamer = 1; + DemoVideoSourceConfig demo = 2; + } + optional uint64 request_async_id = 3; +} +message NewCaptureSourceResponse { required uint64 async_id = 1; } +message NewCaptureSourceCallback { + required uint64 async_id = 1; + oneof message { + string error = 2; + OwnedCaptureSource source = 3; + } +} + +// Start pumping frames from a capture source into its RTC video source. +message StartCaptureRequest { required uint64 capture_handle = 1; } +message StartCaptureResponse { optional string error = 1; } + +// Signal a running capture to stop after the frame in flight. The terminal +// CaptureSourceEvent follows shortly. Stopping an already-finished capture +// is a no-op. +message StopCaptureRequest { required uint64 capture_handle = 1; } +message StopCaptureResponse { optional string error = 1; } + +// Why a capture ended without error. +enum CaptureExit { + // Stopped by StopCaptureRequest (or handle disposal). + CAPTURE_EXIT_STOPPED = 0; + // The producer reached the end of its stream. + CAPTURE_EXIT_END_OF_STREAM = 1; +} + +message CaptureFinished { + required uint64 frames_captured = 1; + required CaptureExit exit = 2; +} + +message CaptureError { required string error = 1; } + +// Delivered exactly once when a started capture ends, regardless of why +// (stop request, end of stream, or failure). +message CaptureSourceEvent { + required uint64 capture_handle = 1; + oneof message { + CaptureFinished finished = 2; + CaptureError error = 3; + } +} diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index b630095e4..2f70fba77 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -27,6 +27,7 @@ import "audio_frame.proto"; import "rpc.proto"; import "data_stream.proto"; import "data_track.proto"; +import "capture.proto"; // **How is the livekit-ffi working: // We refer as the ffi server the Rust server that is running the LiveKit client implementation, and we @@ -179,7 +180,12 @@ message FfiRequest { // Room event ready signal ReadyForRoomEventRequest ready_for_room_event = 83; - // NEXT_ID: 85 + // Capture sources (livekit-capture; requires the `capture` feature) + NewCaptureSourceRequest new_capture_source = 85; + StartCaptureRequest start_capture = 86; + StopCaptureRequest stop_capture = 87; + + // NEXT_ID: 88 } } @@ -304,7 +310,12 @@ message FfiResponse { // Room event ready signal ReadyForRoomEventResponse ready_for_room_event = 82; - // NEXT_ID: 85 + // Capture sources (livekit-capture; requires the `capture` feature) + NewCaptureSourceResponse new_capture_source = 85; + StartCaptureResponse start_capture = 86; + StopCaptureResponse stop_capture = 87; + + // NEXT_ID: 88 } } @@ -369,7 +380,11 @@ message FfiEvent { SimulateScenarioCallback simulate_scenario = 44; - // NEXT_ID: 45 + // Capture sources (livekit-capture; requires the `capture` feature) + NewCaptureSourceCallback new_capture_source = 45; + CaptureSourceEvent capture_source_event = 46; + + // NEXT_ID: 47 } } diff --git a/livekit-ffi/protocol/room.proto b/livekit-ffi/protocol/room.proto index 2494ca2ff..f967f8fec 100644 --- a/livekit-ffi/protocol/room.proto +++ b/livekit-ffi/protocol/room.proto @@ -335,6 +335,8 @@ enum VideoEncoderBackend { ENCODER_BACKEND_NVENC = 3; ENCODER_BACKEND_VAAPI = 4; ENCODER_BACKEND_VIDEOTOOLBOX = 5; + // Pre-encoded passthrough: the application supplies encoded frames. + ENCODER_BACKEND_PRE_ENCODED = 6; } // Controls how the encoder degrades quality when bandwidth is constrained. diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs new file mode 100644 index 000000000..d36ca277a --- /dev/null +++ b/livekit-ffi/src/conversion/capture.rs @@ -0,0 +1,89 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{proto, FfiError, FfiResult}; +use livekit_capture::{ + encoded::EncodedVideoCodec, + primitive::VideoResolution, + sources::gstreamer::{ + GStreamerBitrateUnit, GStreamerRateControlConfig, GStreamerVideoSourceConfig, + }, +}; + +impl From for GStreamerBitrateUnit { + fn from(unit: proto::GstreamerBitrateUnit) -> Self { + match unit { + proto::GstreamerBitrateUnit::Bps => Self::BitsPerSecond, + proto::GstreamerBitrateUnit::Kbps => Self::KilobitsPerSecond, + } + } +} + +pub fn video_codec_from_proto(codec: proto::VideoCodec) -> EncodedVideoCodec { + match codec { + proto::VideoCodec::H264 => EncodedVideoCodec::H264, + proto::VideoCodec::H265 => EncodedVideoCodec::H265, + proto::VideoCodec::Vp8 => EncodedVideoCodec::VP8, + proto::VideoCodec::Vp9 => EncodedVideoCodec::VP9, + proto::VideoCodec::Av1 => EncodedVideoCodec::AV1, + } +} + +pub fn video_codec_to_proto(codec: EncodedVideoCodec) -> Option { + match codec { + EncodedVideoCodec::H264 => Some(proto::VideoCodec::H264), + EncodedVideoCodec::H265 => Some(proto::VideoCodec::H265), + EncodedVideoCodec::VP8 => Some(proto::VideoCodec::Vp8), + EncodedVideoCodec::VP9 => Some(proto::VideoCodec::Vp9), + EncodedVideoCodec::AV1 => Some(proto::VideoCodec::Av1), + // The codec enum is non-exhaustive; codecs unknown to the protocol + // are simply not reported. + _ => None, + } +} + +pub fn gstreamer_config_from_proto( + config: proto::GstreamerVideoSourceConfig, +) -> FfiResult { + let codec = config + .codec + .map(|value| { + proto::VideoCodec::try_from(value) + .map(video_codec_from_proto) + .map_err(|_| FfiError::InvalidRequest("invalid codec".into())) + }) + .transpose()?; + + let rate_control = config + .rate_control + .map(|rate_control| { + let unit = proto::GstreamerBitrateUnit::try_from(rate_control.unit) + .map_err(|_| FfiError::InvalidRequest("invalid bitrate unit".into()))?; + Ok::<_, FfiError>(GStreamerRateControlConfig { + element: rate_control.element, + property: rate_control.property, + unit: unit.into(), + }) + }) + .transpose()?; + + Ok(GStreamerVideoSourceConfig { + pipeline: config.pipeline, + codec, + resolution: config + .resolution + .map(|resolution| VideoResolution::new(resolution.width, resolution.height)), + rate_control, + }) +} diff --git a/livekit-ffi/src/conversion/mod.rs b/livekit-ffi/src/conversion/mod.rs index 364666f24..4ff289823 100644 --- a/livekit-ffi/src/conversion/mod.rs +++ b/livekit-ffi/src/conversion/mod.rs @@ -13,6 +13,8 @@ // limitations under the License. pub mod audio_frame; +#[cfg(feature = "capture")] +pub mod capture; pub mod data_stream; pub mod data_track; pub mod participant; diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs index 2bfb44929..a49c52079 100644 --- a/livekit-ffi/src/conversion/room.rs +++ b/livekit-ffi/src/conversion/room.rs @@ -63,6 +63,9 @@ fn video_encoder_from_proto(backend: Option) -> Option proto::VideoEncoderBackend::EncoderBackendVideotoolbox => { Some(VideoEncoderBackend::VideoToolbox) } + proto::VideoEncoderBackend::EncoderBackendPreEncoded => { + Some(VideoEncoderBackend::PreEncoded) + } } } diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs new file mode 100644 index 000000000..c6a26ed00 --- /dev/null +++ b/livekit-ffi/src/server/capture.rs @@ -0,0 +1,346 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! FFI bindings for livekit-capture sources. +//! +//! A capture source owns its producer (e.g. a GStreamer pipeline) and the +//! pump that feeds an RTC video source, so frames never cross the FFI +//! boundary. The RTC source is exposed as a regular [`FfiVideoSource`], so +//! the existing `CreateVideoTrack`/`PublishTrack` requests work unchanged. + +use livekit_capture::{ + encoded::{EncodedVideoPump, EncodedVideoSource}, + pixel::{PixelVideoPump, PixelVideoSource}, + pump::{PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, + sources::{demo::DemoSource, gstreamer::GStreamerVideoSource}, +}; +use parking_lot::Mutex; + +use super::{video_source::FfiVideoSource, FfiHandle, FfiServer}; +use crate::{ + conversion::capture::{gstreamer_config_from_proto, video_codec_to_proto}, + proto, FfiError, FfiHandleId, FfiResult, +}; + +/// A capture pump of either kind, boxed at the FFI edge. +enum CapturePump { + Pixel(PixelVideoPump>), + Encoded(EncodedVideoPump>), +} + +impl CapturePump { + fn spawn(self) -> std::io::Result { + match self { + Self::Pixel(pump) => pump.spawn(), + Self::Encoded(pump) => pump.spawn(), + } + } +} + +/// State of the capture activity owned by an [`FfiCaptureSource`]. +/// +/// The activity can end without client action (end of stream, error); the +/// FFI object outlives it and is disposed only by the client. +enum CaptureState { + /// Created but not started. + Idle(CapturePump), + /// Started; the watcher task owns the running pump. + Running, + /// The capture ended; the terminal event has been dispatched. + Finished, +} + +pub struct FfiCaptureSource { + pub handle_id: FfiHandleId, + /// Cancellation handle, usable in every state. + stop: PumpStop, + state: Mutex, +} + +impl FfiHandle for FfiCaptureSource {} + +impl Drop for FfiCaptureSource { + fn drop(&mut self) { + // Disposing a running capture stops it; the pump thread observes the + // signal within one bounded wait and drops the source (stopping the + // producer). The watcher task delivers the terminal event, which may + // trail the disposal. + self.stop.stop(); + } +} + +pub fn on_new_capture_source( + server: &'static FfiServer, + request: proto::NewCaptureSourceRequest, +) -> FfiResult { + let async_id = server.resolve_async_id(request.request_async_id); + server.async_runtime.spawn(async move { + let message = match create_capture_source(server, request).await { + Ok(source) => proto::new_capture_source_callback::Message::Source(source), + Err(err) => proto::new_capture_source_callback::Message::Error(err.to_string()), + }; + let _ = server.send_event(proto::ffi_event::Message::NewCaptureSource( + proto::NewCaptureSourceCallback { async_id, message: Some(message) }, + )); + }); + Ok(proto::NewCaptureSourceResponse { async_id }) +} + +async fn create_capture_source( + server: &'static FfiServer, + request: proto::NewCaptureSourceRequest, +) -> FfiResult { + let config = + request.config.ok_or(FfiError::InvalidRequest("missing capture source config".into()))?; + + let pump = match config { + proto::new_capture_source_request::Config::Gstreamer(config) => { + let source = GStreamerVideoSource::new(gstreamer_config_from_proto(config)?) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Encoded(EncodedVideoPump::new(source)) + } + proto::new_capture_source_request::Config::Demo(proto::DemoVideoSourceConfig {}) => { + let source: Box = Box::new(DemoSource::default()); + CapturePump::Pixel(PixelVideoPump::new(source)) + } + }; + + let (kind, resolution, codec, publish_options, rtc_source, stop) = match &pump { + CapturePump::Pixel(pump) => ( + proto::CaptureSourceKind::CaptureSourcePixel, + pump.source().resolution(), + None, + pump.publish_options(), + pump.rtc_source(), + pump.stop_handle(), + ), + CapturePump::Encoded(pump) => ( + proto::CaptureSourceKind::CaptureSourceEncoded, + pump.source().resolution(), + Some(pump.source().codec()), + pump.publish_options(), + pump.rtc_source(), + pump.stop_handle(), + ), + }; + + // The RTC source is a regular client-owned handle, used with the + // existing CreateVideoTrack request. + let source_handle_id = server.next_id(); + let video_source = FfiVideoSource { + handle_id: source_handle_id, + source_type: proto::VideoSourceType::VideoSourceNative, + source: rtc_source, + }; + let video_source_info = proto::VideoSourceInfo::from(&video_source); + server.store_handle(source_handle_id, video_source); + + let info = proto::CaptureSourceInfo { + kind: kind.into(), + resolution: proto::VideoSourceResolution { + width: resolution.width, + height: resolution.height, + }, + codec: codec.and_then(video_codec_to_proto).map(Into::into), + recommended_publish_options: recommended_publish_options_to_proto(&publish_options), + video_source: proto::OwnedVideoSource { + handle: proto::FfiOwnedHandle { id: source_handle_id }, + info: video_source_info, + }, + }; + + let capture_handle_id = server.next_id(); + server.store_handle( + capture_handle_id, + FfiCaptureSource { + handle_id: capture_handle_id, + stop, + state: Mutex::new(CaptureState::Idle(pump)), + }, + ); + + Ok(proto::OwnedCaptureSource { + handle: proto::FfiOwnedHandle { id: capture_handle_id }, + info, + }) +} + +/// Maps the pump-derived publish options into the proto options the client +/// merges its own settings over. +fn recommended_publish_options_to_proto( + options: &livekit::options::TrackPublishOptions, +) -> proto::TrackPublishOptions { + use livekit::options::VideoCodec; + let video_codec = match options.video_codec { + VideoCodec::VP8 => proto::VideoCodec::Vp8, + VideoCodec::H264 => proto::VideoCodec::H264, + VideoCodec::AV1 => proto::VideoCodec::Av1, + VideoCodec::VP9 => proto::VideoCodec::Vp9, + VideoCodec::H265 => proto::VideoCodec::H265, + }; + let video_encoder = match options.video_encoder { + livekit::options::VideoEncoderBackend::PreEncoded => { + Some(proto::VideoEncoderBackend::EncoderBackendPreEncoded.into()) + } + _ => None, + }; + proto::TrackPublishOptions { + video_codec: Some(video_codec.into()), + video_encoder, + simulcast: Some(options.simulcast), + ..Default::default() + } +} + +pub fn on_start_capture( + server: &'static FfiServer, + request: proto::StartCaptureRequest, +) -> FfiResult { + let capture_handle = request.capture_handle; + let ffi_capture = server.retrieve_handle::(capture_handle)?; + + let mut state = ffi_capture.state.lock(); + let pump = match std::mem::replace(&mut *state, CaptureState::Running) { + CaptureState::Idle(pump) => pump, + other => { + let error = match &other { + CaptureState::Running => "capture is already started", + _ => "capture has already finished", + }; + *state = other; + return Ok(proto::StartCaptureResponse { error: Some(error.to_owned()) }); + } + }; + + let running = match pump.spawn() { + Ok(running) => running, + Err(err) => { + *state = CaptureState::Finished; + return Ok(proto::StartCaptureResponse { + error: Some(format!("failed to start capture: {err}")), + }); + } + }; + drop(state); + drop(ffi_capture); + + // The watcher owns the running pump and delivers the terminal event + // exactly once, whether the capture is stopped, ends, or fails. + server.async_runtime.spawn(async move { + let result = running.join_async().await; + if let Ok(ffi_capture) = server.retrieve_handle::(capture_handle) { + *ffi_capture.state.lock() = CaptureState::Finished; + } + let _ = server.send_event(proto::ffi_event::Message::CaptureSourceEvent( + proto::CaptureSourceEvent { + capture_handle, + message: Some(capture_result_to_proto(result)), + }, + )); + }); + + Ok(proto::StartCaptureResponse { error: None }) +} + +fn capture_result_to_proto( + result: Result, +) -> proto::capture_source_event::Message { + match result { + Ok(stats) => { + let exit = match stats.exit { + PumpExit::Stopped => proto::CaptureExit::Stopped, + PumpExit::EndOfStream => proto::CaptureExit::EndOfStream, + }; + proto::capture_source_event::Message::Finished(proto::CaptureFinished { + frames_captured: stats.frames_captured, + exit: exit.into(), + }) + } + Err(err) => proto::capture_source_event::Message::Error(proto::CaptureError { + error: err.to_string(), + }), + } +} + +pub fn on_stop_capture( + server: &'static FfiServer, + request: proto::StopCaptureRequest, +) -> FfiResult { + let ffi_capture = server.retrieve_handle::(request.capture_handle)?; + ffi_capture.stop.stop(); + Ok(proto::StopCaptureResponse { error: None }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::FFI_SERVER; + use std::time::Duration; + + fn server() -> &'static FfiServer { + &FFI_SERVER + } + + #[test] + fn demo_capture_lifecycle() { + let request = proto::NewCaptureSourceRequest { + config: Some(proto::new_capture_source_request::Config::Demo( + proto::DemoVideoSourceConfig {}, + )), + request_async_id: None, + }; + let source = server() + .async_runtime + .block_on(create_capture_source(server(), request)) + .expect("demo capture source should build"); + assert_eq!(source.info.kind(), proto::CaptureSourceKind::CaptureSourcePixel); + assert_eq!(source.info.resolution.width, 1280); + let capture_handle = source.handle.id; + + // Stopping before starting is allowed; the pump then exits + // immediately once started, and the watcher marks it finished. + let response = on_stop_capture( + server(), + proto::StopCaptureRequest { capture_handle }, + ) + .unwrap(); + assert_eq!(response.error, None); + + let response = + on_start_capture(server(), proto::StartCaptureRequest { capture_handle }).unwrap(); + assert_eq!(response.error, None); + + let response = + on_start_capture(server(), proto::StartCaptureRequest { capture_handle }).unwrap(); + assert!(response.error.is_some(), "double start must be rejected"); + + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + { + let ffi_capture = + server().retrieve_handle::(capture_handle).unwrap(); + if matches!(*ffi_capture.state.lock(), CaptureState::Finished) { + break; + } + } + assert!(std::time::Instant::now() < deadline, "capture did not finish"); + std::thread::sleep(Duration::from_millis(10)); + } + + server().drop_handle(capture_handle); + server().drop_handle(source.info.video_source.handle.id); + } +} diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index 1689bdbbe..131fda19a 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -36,6 +36,8 @@ use crate::{proto, proto::FfiEvent, FfiError, FfiHandleId, FfiResult, INVALID_HA pub mod audio_plugin; pub mod audio_source; pub mod audio_stream; +#[cfg(feature = "capture")] +pub mod capture; pub mod colorcvt; pub mod data_stream; pub mod data_track; diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index bb94f6232..393d52314 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -23,6 +23,8 @@ use livekit::{ }; use parking_lot::Mutex; +#[cfg(feature = "capture")] +use super::capture; use super::{ audio_source, audio_stream, colorcvt, data_stream, data_track, participant::FfiParticipant, @@ -1440,6 +1442,19 @@ pub fn handle_request( } Request::StartRecording(req) => platform_audio::on_start_recording(server, req)?.into(), Request::StopRecording(req) => platform_audio::on_stop_recording(server, req)?.into(), + + #[cfg(feature = "capture")] + Request::NewCaptureSource(req) => capture::on_new_capture_source(server, req)?.into(), + #[cfg(feature = "capture")] + Request::StartCapture(req) => capture::on_start_capture(server, req)?.into(), + #[cfg(feature = "capture")] + Request::StopCapture(req) => capture::on_stop_capture(server, req)?.into(), + #[cfg(not(feature = "capture"))] + Request::NewCaptureSource(_) | Request::StartCapture(_) | Request::StopCapture(_) => { + return Err(FfiError::InvalidRequest( + "livekit-ffi was built without the 'capture' feature".into(), + )); + } }); Ok(res) From c6142dcc872e6554c0c534c3bab9bc87f76a076e Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:31:45 +0000 Subject: [PATCH 31/56] generated protobuf --- livekit-ffi-node-bindings/proto/ffi_pb.d.ts | 55 ++++++++++++++++++++ livekit-ffi-node-bindings/proto/ffi_pb.js | 9 ++++ livekit-ffi-node-bindings/proto/room_pb.d.ts | 7 +++ livekit-ffi-node-bindings/proto/room_pb.js | 1 + 4 files changed, 72 insertions(+) diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts index ac81cb93e..12cb91bbe 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts @@ -28,6 +28,7 @@ import type { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, Registe import type { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } from "./track_publication_pb.js"; import type { ByteStreamOpenCallback, ByteStreamOpenRequest, ByteStreamOpenResponse, ByteStreamReaderEvent, ByteStreamReaderReadAllCallback, ByteStreamReaderReadAllRequest, ByteStreamReaderReadAllResponse, ByteStreamReaderReadIncrementalRequest, ByteStreamReaderReadIncrementalResponse, ByteStreamReaderWriteToFileCallback, ByteStreamReaderWriteToFileRequest, ByteStreamReaderWriteToFileResponse, ByteStreamWriterCloseCallback, ByteStreamWriterCloseRequest, ByteStreamWriterCloseResponse, ByteStreamWriterWriteCallback, ByteStreamWriterWriteRequest, ByteStreamWriterWriteResponse, StreamSendBytesCallback, StreamSendBytesRequest, StreamSendBytesResponse, StreamSendFileCallback, StreamSendFileRequest, StreamSendFileResponse, StreamSendTextCallback, StreamSendTextRequest, StreamSendTextResponse, TextStreamOpenCallback, TextStreamOpenRequest, TextStreamOpenResponse, TextStreamReaderEvent, TextStreamReaderReadAllCallback, TextStreamReaderReadAllRequest, TextStreamReaderReadAllResponse, TextStreamReaderReadIncrementalRequest, TextStreamReaderReadIncrementalResponse, TextStreamWriterCloseCallback, TextStreamWriterCloseRequest, TextStreamWriterCloseResponse, TextStreamWriterWriteCallback, TextStreamWriterWriteRequest, TextStreamWriterWriteResponse } from "./data_stream_pb.js"; import type { DataTrackStreamEvent, DataTrackStreamReadRequest, DataTrackStreamReadResponse, DefineSchemaCallback, DefineSchemaRequest, DefineSchemaResponse, GetSchemaCallback, GetSchemaRequest, GetSchemaResponse, LocalDataTrackIsPublishedRequest, LocalDataTrackIsPublishedResponse, LocalDataTrackTryPushRequest, LocalDataTrackTryPushResponse, LocalDataTrackUnpublishRequest, LocalDataTrackUnpublishResponse, PublishDataTrackCallback, PublishDataTrackRequest, PublishDataTrackResponse, RemoteDataTrackIsPublishedRequest, RemoteDataTrackIsPublishedResponse, RemoteDataTrackSetPipelineOptionsRequest, RemoteDataTrackSetPipelineOptionsResponse, SubscribeDataTrackRequest, SubscribeDataTrackResponse } from "./data_track_pb.js"; +import type { CaptureSourceEvent, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } from "./capture_pb.js"; /** * @generated from enum livekit.proto.LogLevel @@ -611,6 +612,26 @@ export declare class FfiRequest extends Message { */ value: ReadyForRoomEventRequest; case: "readyForRoomEvent"; + } | { + /** + * Capture sources (livekit-capture; requires the `capture` feature) + * + * @generated from field: livekit.proto.NewCaptureSourceRequest new_capture_source = 87; + */ + value: NewCaptureSourceRequest; + case: "newCaptureSource"; + } | { + /** + * @generated from field: livekit.proto.StartCaptureRequest start_capture = 88; + */ + value: StartCaptureRequest; + case: "startCapture"; + } | { + /** + * @generated from field: livekit.proto.StopCaptureRequest stop_capture = 89; + */ + value: StopCaptureRequest; + case: "stopCapture"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); @@ -1173,6 +1194,26 @@ export declare class FfiResponse extends Message { */ value: ReadyForRoomEventResponse; case: "readyForRoomEvent"; + } | { + /** + * Capture sources (livekit-capture; requires the `capture` feature) + * + * @generated from field: livekit.proto.NewCaptureSourceResponse new_capture_source = 87; + */ + value: NewCaptureSourceResponse; + case: "newCaptureSource"; + } | { + /** + * @generated from field: livekit.proto.StartCaptureResponse start_capture = 88; + */ + value: StartCaptureResponse; + case: "startCapture"; + } | { + /** + * @generated from field: livekit.proto.StopCaptureResponse stop_capture = 89; + */ + value: StopCaptureResponse; + case: "stopCapture"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); @@ -1481,6 +1522,20 @@ export declare class FfiEvent extends Message { */ value: GetSchemaCallback; case: "getSchema"; + } | { + /** + * Capture sources (livekit-capture; requires the `capture` feature) + * + * @generated from field: livekit.proto.NewCaptureSourceCallback new_capture_source = 47; + */ + value: NewCaptureSourceCallback; + case: "newCaptureSource"; + } | { + /** + * @generated from field: livekit.proto.CaptureSourceEvent capture_source_event = 48; + */ + value: CaptureSourceEvent; + case: "captureSourceEvent"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.js b/livekit-ffi-node-bindings/proto/ffi_pb.js index 4f0bdd422..c61a091f6 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.js +++ b/livekit-ffi-node-bindings/proto/ffi_pb.js @@ -30,6 +30,7 @@ const { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMe const { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } = require("./track_publication_pb.js"); const { ByteStreamOpenCallback, ByteStreamOpenRequest, ByteStreamOpenResponse, ByteStreamReaderEvent, ByteStreamReaderReadAllCallback, ByteStreamReaderReadAllRequest, ByteStreamReaderReadAllResponse, ByteStreamReaderReadIncrementalRequest, ByteStreamReaderReadIncrementalResponse, ByteStreamReaderWriteToFileCallback, ByteStreamReaderWriteToFileRequest, ByteStreamReaderWriteToFileResponse, ByteStreamWriterCloseCallback, ByteStreamWriterCloseRequest, ByteStreamWriterCloseResponse, ByteStreamWriterWriteCallback, ByteStreamWriterWriteRequest, ByteStreamWriterWriteResponse, StreamSendBytesCallback, StreamSendBytesRequest, StreamSendBytesResponse, StreamSendFileCallback, StreamSendFileRequest, StreamSendFileResponse, StreamSendTextCallback, StreamSendTextRequest, StreamSendTextResponse, TextStreamOpenCallback, TextStreamOpenRequest, TextStreamOpenResponse, TextStreamReaderEvent, TextStreamReaderReadAllCallback, TextStreamReaderReadAllRequest, TextStreamReaderReadAllResponse, TextStreamReaderReadIncrementalRequest, TextStreamReaderReadIncrementalResponse, TextStreamWriterCloseCallback, TextStreamWriterCloseRequest, TextStreamWriterCloseResponse, TextStreamWriterWriteCallback, TextStreamWriterWriteRequest, TextStreamWriterWriteResponse } = require("./data_stream_pb.js"); const { DataTrackStreamEvent, DataTrackStreamReadRequest, DataTrackStreamReadResponse, DefineSchemaCallback, DefineSchemaRequest, DefineSchemaResponse, GetSchemaCallback, GetSchemaRequest, GetSchemaResponse, LocalDataTrackIsPublishedRequest, LocalDataTrackIsPublishedResponse, LocalDataTrackTryPushRequest, LocalDataTrackTryPushResponse, LocalDataTrackUnpublishRequest, LocalDataTrackUnpublishResponse, PublishDataTrackCallback, PublishDataTrackRequest, PublishDataTrackResponse, RemoteDataTrackIsPublishedRequest, RemoteDataTrackIsPublishedResponse, RemoteDataTrackSetPipelineOptionsRequest, RemoteDataTrackSetPipelineOptionsResponse, SubscribeDataTrackRequest, SubscribeDataTrackResponse } = require("./data_track_pb.js"); +const { CaptureSourceEvent, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } = require("./capture_pb.js"); /** * @generated from enum livekit.proto.LogLevel @@ -139,6 +140,9 @@ const FfiRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 81, name: "start_recording", kind: "message", T: StartRecordingRequest, oneof: "message" }, { no: 82, name: "stop_recording", kind: "message", T: StopRecordingRequest, oneof: "message" }, { no: 83, name: "ready_for_room_event", kind: "message", T: ReadyForRoomEventRequest, oneof: "message" }, + { no: 87, name: "new_capture_source", kind: "message", T: NewCaptureSourceRequest, oneof: "message" }, + { no: 88, name: "start_capture", kind: "message", T: StartCaptureRequest, oneof: "message" }, + { no: 89, name: "stop_capture", kind: "message", T: StopCaptureRequest, oneof: "message" }, ], ); @@ -234,6 +238,9 @@ const FfiResponse = /*@__PURE__*/ proto2.makeMessageType( { no: 80, name: "start_recording", kind: "message", T: StartRecordingResponse, oneof: "message" }, { no: 81, name: "stop_recording", kind: "message", T: StopRecordingResponse, oneof: "message" }, { no: 82, name: "ready_for_room_event", kind: "message", T: ReadyForRoomEventResponse, oneof: "message" }, + { no: 87, name: "new_capture_source", kind: "message", T: NewCaptureSourceResponse, oneof: "message" }, + { no: 88, name: "start_capture", kind: "message", T: StartCaptureResponse, oneof: "message" }, + { no: 89, name: "stop_capture", kind: "message", T: StopCaptureResponse, oneof: "message" }, ], ); @@ -292,6 +299,8 @@ const FfiEvent = /*@__PURE__*/ proto2.makeMessageType( { no: 44, name: "simulate_scenario", kind: "message", T: SimulateScenarioCallback, oneof: "message" }, { no: 45, name: "define_schema", kind: "message", T: DefineSchemaCallback, oneof: "message" }, { no: 46, name: "get_schema", kind: "message", T: GetSchemaCallback, oneof: "message" }, + { no: 47, name: "new_capture_source", kind: "message", T: NewCaptureSourceCallback, oneof: "message" }, + { no: 48, name: "capture_source_event", kind: "message", T: CaptureSourceEvent, oneof: "message" }, ], ); diff --git a/livekit-ffi-node-bindings/proto/room_pb.d.ts b/livekit-ffi-node-bindings/proto/room_pb.d.ts index c29775d69..725466dea 100644 --- a/livekit-ffi-node-bindings/proto/room_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/room_pb.d.ts @@ -125,6 +125,13 @@ export declare enum VideoEncoderBackend { * @generated from enum value: ENCODER_BACKEND_VIDEOTOOLBOX = 5; */ ENCODER_BACKEND_VIDEOTOOLBOX = 5, + + /** + * Pre-encoded passthrough: the application supplies encoded frames. + * + * @generated from enum value: ENCODER_BACKEND_PRE_ENCODED = 6; + */ + ENCODER_BACKEND_PRE_ENCODED = 6, } /** diff --git a/livekit-ffi-node-bindings/proto/room_pb.js b/livekit-ffi-node-bindings/proto/room_pb.js index ea71ea114..03c23527d 100644 --- a/livekit-ffi-node-bindings/proto/room_pb.js +++ b/livekit-ffi-node-bindings/proto/room_pb.js @@ -65,6 +65,7 @@ const VideoEncoderBackend = /*@__PURE__*/ proto2.makeEnum( {no: 3, name: "ENCODER_BACKEND_NVENC"}, {no: 4, name: "ENCODER_BACKEND_VAAPI"}, {no: 5, name: "ENCODER_BACKEND_VIDEOTOOLBOX"}, + {no: 6, name: "ENCODER_BACKEND_PRE_ENCODED"}, ], ); From 33e7b3d663f2f259a62fc0907c8d7ab159f840c0 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:30:51 -0700 Subject: [PATCH 32/56] Add serde and schemars features --- Cargo.lock | 58 +++++++++++++++++++++ Cargo.toml | 1 + livekit-capture/Cargo.toml | 10 ++++ livekit-capture/src/encoded/mod.rs | 6 +++ livekit-capture/src/lib.rs | 2 - livekit-capture/src/primitive.rs | 6 +++ livekit-capture/src/sources/demo.rs | 66 ++++++++++++++++-------- livekit-capture/src/sources/gstreamer.rs | 23 ++++++++- 8 files changed, 147 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0944e45bc..0ad050a63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4163,6 +4163,8 @@ dependencies = [ "gstreamer-app", "livekit", "log", + "schemars", + "serde", "thiserror 2.0.19", "tokio", ] @@ -6479,6 +6481,26 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "regex" version = "1.13.1" @@ -6811,6 +6833,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.3", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -6956,6 +7003,17 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_json" version = "1.0.151" diff --git a/Cargo.toml b/Cargo.toml index 398b49449..9c74f2d39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ prost = "0.14" prost-build = "0.14" prost-types = "0.14" rand = "0.9" +schemars = "1.2" serde = "1" serde_json = "1.0" thiserror = "2" diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 0e79bc916..33f41407b 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -13,6 +13,8 @@ gstreamer = { version = "0.25.2", optional = true } gstreamer-app = { version = "0.25.2", optional = true } livekit = { workspace = true } log = { workspace = true } +schemars = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"], optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["sync"] } @@ -27,6 +29,14 @@ default = ["demo", "gstreamer"] # TODO: Remove after testing # `new_blocking` constructors are available. tokio = ["tokio/rt"] +# Serde derives on the source configuration types, for applications that +# build sources from a configuration file rather than from code. +serde = ["dep:serde"] + +# JSON Schema derives on the same types. The schema is generated from the +# serde attributes, so it would not describe the wire format without them. +schemars = ["dep:schemars", "serde"] + # Pixel sources demo = [] diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index 2f3855fc1..88c05f347 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -106,6 +106,12 @@ pub enum EncodedWireFormat { /// Encoded video codec carried by an [`EncodedAccessUnit`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "lowercase") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] #[non_exhaustive] pub enum EncodedVideoCodec { /// H.264/AVC video. diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 1bae9f094..7d0c62f12 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Capture sources and helpers for publishing video with LiveKit. - pub mod encoded; pub mod error; pub mod pixel; diff --git a/livekit-capture/src/primitive.rs b/livekit-capture/src/primitive.rs index 0fb599d65..60251c421 100644 --- a/livekit-capture/src/primitive.rs +++ b/livekit-capture/src/primitive.rs @@ -24,6 +24,12 @@ /// Pixel dimensions of a video frame. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct VideoResolution { /// Frame width in pixels. pub width: u32, diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index 5dad9b8d1..2a71a4196 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -20,6 +20,10 @@ use std::{ thread, time::{Duration, Instant}, }; +use thiserror::Error; + +/// How long each palette color is shown before cycling to the next. +const COLOR_INTERVAL: Duration = Duration::from_millis(500); /// Colors the demo source cycles through, as `(r, g, b)`. const PALETTE: [(u8, u8, u8); 6] = [ @@ -33,22 +37,22 @@ const PALETTE: [(u8, u8, u8); 6] = [ /// Configuration for a [`DemoSource`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(default, deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct DemoSourceConfig { /// Output resolution. pub resolution: VideoResolution, /// Output frame rate in frames per second. pub framerate_fps: u32, - /// How long each palette color is shown before cycling to the next. - pub color_interval: Duration, } impl Default for DemoSourceConfig { fn default() -> Self { - Self { - resolution: VideoResolution { width: 1280, height: 720 }, - framerate_fps: 30, - color_interval: Duration::from_millis(500), - } + Self { resolution: VideoResolution { width: 1280, height: 720 }, framerate_fps: 30 } } } @@ -73,19 +77,24 @@ impl DemoSource { /// /// # Panics /// - /// Panics if the configured resolution or frame rate is zero, or if the - /// color interval is shorter than one frame. + /// Panics if the configuration is invalid; use [`DemoSource::try_new`] + /// for configuration that is not known good at compile time. pub fn new(config: DemoSourceConfig) -> Self { + Self::try_new(config).expect("demo source configuration is valid") + } + + /// Creates a demo source, rejecting an invalid configuration. + pub fn try_new(config: DemoSourceConfig) -> Result { let VideoResolution { width, height } = config.resolution; - assert!(width > 0 && height > 0, "demo source resolution must be non-zero"); - assert!(config.framerate_fps > 0, "demo source frame rate must be non-zero"); - assert!( - config.color_interval >= Duration::from_secs(1) / config.framerate_fps, - "demo source color interval must be at least one frame" - ); + if width == 0 || height == 0 { + return Err(SourceError::new(DemoSourceConfigError::ZeroResolution)); + } + if config.framerate_fps == 0 { + return Err(SourceError::new(DemoSourceConfigError::ZeroFramerate)); + } let colors = PALETTE.iter().map(|&color| yuv_from_rgb(color)).collect(); - Self { config, colors, started: None, frame_index: 0 } + Ok(Self { config, colors, started: None, frame_index: 0 }) } fn frame_interval(&self) -> Duration { @@ -93,6 +102,17 @@ impl DemoSource { } } +/// Error returned when a [`DemoSourceConfig`] cannot produce frames. +#[derive(Debug, Error)] +pub enum DemoSourceConfigError { + /// The configured resolution has a zero component. + #[error("demo source resolution must be non-zero")] + ZeroResolution, + /// The configured frame rate is zero. + #[error("demo source frame rate must be non-zero")] + ZeroFramerate, +} + impl Default for DemoSource { fn default() -> Self { Self::new(DemoSourceConfig::default()) @@ -118,8 +138,7 @@ impl PixelVideoSource for DemoSource { } let timestamp_us = elapsed.as_micros() as i64; - let color_index = - (elapsed.as_micros() / self.config.color_interval.as_micros().max(1)) as usize; + let color_index = (elapsed.as_micros() / COLOR_INTERVAL.as_micros()) as usize; let (y, u, v) = self.colors[color_index % self.colors.len()]; self.frame_index += 1; @@ -156,7 +175,6 @@ mod tests { DemoSourceConfig { resolution: VideoResolution { width: 64, height: 36 }, framerate_fps: 1000, - color_interval: Duration::from_millis(2), } } @@ -186,11 +204,17 @@ mod tests { #[test] fn colors_cycle_at_the_color_interval() { - let mut source = DemoSource::new(test_config()); + // Two frames per color, so the first boundary lands on frame three. + // The source paces itself in real time, so this trades frame count + // for the ~COLOR_INTERVAL the test spends sleeping either way. + let frame_interval = COLOR_INTERVAL / 2; + let mut source = DemoSource::new(DemoSourceConfig { + resolution: VideoResolution { width: 64, height: 36 }, + framerate_fps: (Duration::from_secs(1).as_micros() / frame_interval.as_micros()) as u32, + }); let luma = |frame: &BoxVideoFrame| frame.buffer.as_i420().unwrap().data().0[0]; - // Two frames per color at 1000 fps with a 2 ms interval. let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); let same_color = source.next_frame(&PumpStop::new()).unwrap().unwrap(); let next_color = source.next_frame(&PumpStop::new()).unwrap().unwrap(); diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 86074b826..98f28638d 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -65,6 +65,12 @@ impl GStreamerSampleFormat { /// Configuration for a GStreamer encoded video source. #[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct GStreamerVideoSourceConfig { /// GStreamer launch description for the encoded producer pipeline. /// @@ -73,24 +79,33 @@ pub struct GStreamerVideoSourceConfig { pub pipeline: String, /// Codec expected from the pipeline; inferred from pipeline caps when - /// `None`. + /// omitted. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] pub codec: Option, /// Encoded frame resolution. /// - /// When `None`, the resolution is discovered from the first sample's + /// When omitted, the resolution is discovered from the first sample's /// negotiated caps — construction then waits for the pipeline to produce /// data. When set, construction returns without waiting, and the first /// sample is verified against the declared resolution. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] pub resolution: Option, /// Forwards WebRTC rate-control targets to an encoder element's bitrate /// property. Without this, the pipeline encodes at a fixed bitrate. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] pub rate_control: Option, } /// Binding from WebRTC rate-control targets to a GStreamer encoder property. #[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct GStreamerRateControlConfig { /// Name of the encoder element in the pipeline (e.g. `lk_encoder`). pub element: String, @@ -105,10 +120,14 @@ pub struct GStreamerRateControlConfig { /// Bitrate unit used by a GStreamer encoder property. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub enum GStreamerBitrateUnit { /// The encoder property expects bits per second. + #[cfg_attr(feature = "serde", serde(rename = "bps"))] BitsPerSecond, /// The encoder property expects kilobits per second. + #[cfg_attr(feature = "serde", serde(rename = "kbps"))] KilobitsPerSecond, } From 56ac2c9b0ea44a33ff49e3a5a51fc350399a9d9d Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:00:35 -0700 Subject: [PATCH 33/56] Uniform source construction - No default impls for config - Uniform constructor naming --- livekit-capture/src/sources/demo.rs | 33 +++++---------------------- livekit-capture/src/sources/mod.rs | 2 -- livekit-ffi/protocol/capture.proto | 7 +++++- livekit-ffi/src/conversion/capture.rs | 21 +++++++++++++---- livekit-ffi/src/server/capture.rs | 11 ++++++--- 5 files changed, 36 insertions(+), 38 deletions(-) diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index 2a71a4196..c9c0934dd 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -40,7 +40,7 @@ const PALETTE: [(u8, u8, u8); 6] = [ #[cfg_attr( feature = "serde", derive(serde::Serialize, serde::Deserialize), - serde(default, deny_unknown_fields) + serde(deny_unknown_fields) )] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct DemoSourceConfig { @@ -50,12 +50,6 @@ pub struct DemoSourceConfig { pub framerate_fps: u32, } -impl Default for DemoSourceConfig { - fn default() -> Self { - Self { resolution: VideoResolution { width: 1280, height: 720 }, framerate_fps: 30 } - } -} - /// Pixel video source that produces solid-color frames, cycling through a /// fixed palette. /// @@ -73,18 +67,8 @@ pub struct DemoSource { } impl DemoSource { - /// Creates a demo source. - /// - /// # Panics - /// - /// Panics if the configuration is invalid; use [`DemoSource::try_new`] - /// for configuration that is not known good at compile time. - pub fn new(config: DemoSourceConfig) -> Self { - Self::try_new(config).expect("demo source configuration is valid") - } - /// Creates a demo source, rejecting an invalid configuration. - pub fn try_new(config: DemoSourceConfig) -> Result { + pub fn new(config: DemoSourceConfig) -> Result { let VideoResolution { width, height } = config.resolution; if width == 0 || height == 0 { return Err(SourceError::new(DemoSourceConfigError::ZeroResolution)); @@ -113,12 +97,6 @@ pub enum DemoSourceConfigError { ZeroFramerate, } -impl Default for DemoSource { - fn default() -> Self { - Self::new(DemoSourceConfig::default()) - } -} - impl PixelVideoSource for DemoSource { fn resolution(&self) -> VideoResolution { self.config.resolution @@ -180,7 +158,7 @@ mod tests { #[test] fn yields_frames_with_configured_dimensions() { - let mut source = DemoSource::new(test_config()); + let mut source = DemoSource::new(test_config()).unwrap(); let frame = source.next_frame(&PumpStop::new()).unwrap().unwrap(); assert_eq!((frame.buffer.width(), frame.buffer.height()), (64, 36)); @@ -194,7 +172,7 @@ mod tests { #[test] fn timestamps_follow_the_frame_rate() { - let mut source = DemoSource::new(test_config()); + let mut source = DemoSource::new(test_config()).unwrap(); let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); let second = source.next_frame(&PumpStop::new()).unwrap().unwrap(); @@ -211,7 +189,8 @@ mod tests { let mut source = DemoSource::new(DemoSourceConfig { resolution: VideoResolution { width: 64, height: 36 }, framerate_fps: (Duration::from_secs(1).as_micros() / frame_interval.as_micros()) as u32, - }); + }) + .unwrap(); let luma = |frame: &BoxVideoFrame| frame.buffer.as_i420().unwrap().data().0[0]; diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index f5529ea7b..017f27e44 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Optional capture sources that feed the shared capture paths. - #[cfg(feature = "demo")] pub mod demo; diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 3c391e08f..8ab77c595 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -73,7 +73,12 @@ message GstreamerVideoSourceConfig { } // Test source producing solid-color frames, cycling through a palette. -message DemoVideoSourceConfig {} +message DemoVideoSourceConfig { + // Output resolution. + required VideoSourceResolution resolution = 1; + // Output frame rate in frames per second. + required uint32 framerate_fps = 2; +} // Kind of media a capture source produces. enum CaptureSourceKind { diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index d36ca277a..7672e7492 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -16,11 +16,24 @@ use crate::{proto, FfiError, FfiResult}; use livekit_capture::{ encoded::EncodedVideoCodec, primitive::VideoResolution, - sources::gstreamer::{ - GStreamerBitrateUnit, GStreamerRateControlConfig, GStreamerVideoSourceConfig, + sources::{ + demo::DemoSourceConfig, + gstreamer::{GStreamerBitrateUnit, GStreamerRateControlConfig, GStreamerVideoSourceConfig}, }, }; +impl From for VideoResolution { + fn from(resolution: proto::VideoSourceResolution) -> Self { + Self::new(resolution.width, resolution.height) + } +} + +impl From for DemoSourceConfig { + fn from(config: proto::DemoVideoSourceConfig) -> Self { + Self { resolution: config.resolution.into(), framerate_fps: config.framerate_fps } + } +} + impl From for GStreamerBitrateUnit { fn from(unit: proto::GstreamerBitrateUnit) -> Self { match unit { @@ -81,9 +94,7 @@ pub fn gstreamer_config_from_proto( Ok(GStreamerVideoSourceConfig { pipeline: config.pipeline, codec, - resolution: config - .resolution - .map(|resolution| VideoResolution::new(resolution.width, resolution.height)), + resolution: config.resolution.map(VideoResolution::from), rate_control, }) } diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index c6a26ed00..97892bc83 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -112,8 +112,10 @@ async fn create_capture_source( let source: Box = Box::new(source); CapturePump::Encoded(EncodedVideoPump::new(source)) } - proto::new_capture_source_request::Config::Demo(proto::DemoVideoSourceConfig {}) => { - let source: Box = Box::new(DemoSource::default()); + proto::new_capture_source_request::Config::Demo(config) => { + let source = DemoSource::new(config.into()) + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); CapturePump::Pixel(PixelVideoPump::new(source)) } }; @@ -298,7 +300,10 @@ mod tests { fn demo_capture_lifecycle() { let request = proto::NewCaptureSourceRequest { config: Some(proto::new_capture_source_request::Config::Demo( - proto::DemoVideoSourceConfig {}, + proto::DemoVideoSourceConfig { + resolution: proto::VideoSourceResolution { width: 1280, height: 720 }, + framerate_fps: 30, + }, )), request_async_id: None, }; From 6c6c62611a48eb3da226b3ed6a6be7e1ee6aa0d9 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:46:22 -0700 Subject: [PATCH 34/56] Improve documentation --- livekit-capture/Cargo.toml | 15 ++---- livekit-capture/README.md | 106 +++++++++++++++++++------------------ 2 files changed, 59 insertions(+), 62 deletions(-) diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 33f41407b..ee10daa9b 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -23,20 +23,13 @@ tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] default = ["demo", "gstreamer"] # TODO: Remove after testing - -# Async source constructors (`new`), which run blocking construction and -# stream discovery on the tokio blocking pool. Without this feature only the -# `new_blocking` constructors are available. -tokio = ["tokio/rt"] - -# Serde derives on the source configuration types, for applications that -# build sources from a configuration file rather than from code. serde = ["dep:serde"] - -# JSON Schema derives on the same types. The schema is generated from the -# serde attributes, so it would not describe the wire format without them. schemars = ["dep:schemars", "serde"] +# Async convenience wrappers for blocking source construction when using +# a tokio runtime. +tokio = ["tokio/rt"] + # Pixel sources demo = [] diff --git a/livekit-capture/README.md b/livekit-capture/README.md index da90af278..82f48f44c 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -1,51 +1,55 @@ -# livekit-capture - -Capture sources and helpers for publishing video with the LiveKit Rust SDK. -The optional `gstreamer` feature turns a GStreamer `appsink` into an encoded -ingest source; the `demo` feature adds a synthetic pixel source for testing. - -## Library entry points - -- `pixel::PixelVideoSource` and `encoded::EncodedVideoSource` — the traits a - capture backend implements: pixel sources yield libwebrtc `VideoFrame`s - (any `VideoBuffer`, CPU or native, with no intermediate copy) published - through the WebRTC encoder; encoded sources produce crate-owned access - units published as passthrough. Both traits are object-safe and - implemented for `Box`, so sources can be constructed dynamically - and driven through the same pumps. The crate owns a type only where it - adds semantics (`encoded::EncodedAccessUnit` and the parsing/validation - vocabulary); elsewhere livekit's types are used directly. -- `pixel::PixelVideoPump` and `encoded::EncodedVideoPump` — bridge a - source into a publishable RTC track: each builds the matching - `NativeVideoSource`, derives publish options (`EncodedVideoPump` selects - the passthrough encoder), and runs the capture loop on a plain thread. - Encoded pumps forward downstream keyframe and rate-control requests back - to the source and drop pre-roll deltas until the first keyframe. Both - spawn into the same `pump::RunningPump`, so an application supervises - running pumps of either kind uniformly (`stop()`, `join_async()`, stats); - the `pump` module holds this shared machinery. -- `sources::gstreamer::GStreamerVideoSource` — built solely from - configuration (`GStreamerVideoSourceConfig`: launch description, plus - optional codec, resolution, and rate-control binding). The source owns its - pipeline: it is started at construction, construction fails loudly on - pipeline problems, bus errors surface as source errors, and the pipeline - stops when the source is dropped. Codec and resolution are discovered from - pipeline caps when omitted (a declared resolution skips the discovery wait - and is verified against the stream); a mid-stream caps change is an error - until track republication is supported. With the `tokio` crate feature, - `new` runs construction and discovery on the blocking pool — the - convention for all backends — while `new_blocking` serves non-async - consumers. `encoded_caps_string` remains the single per-codec caps table - for writing producer pipelines. - -## GStreamer ingest - -`GStreamerVideoSource` implements `EncodedVideoSource` on top of a pipeline -whose `appsink` (named `lk_appsink`, or attached automatically to one -unlinked encoded pad) produces H.264 (Annex-B or AVC), H.265 Annex-B, VP8, -VP9, or AV1 access units. Drive it with an `EncodedVideoPump`, which builds -the encoded RTC source, derives the passthrough publish options, and -forwards keyframe requests (answered with a `GstForceKeyUnit` upstream -event) and rate-control targets back to the pipeline. Passthrough is -single-layer (`L1T1`); access units carrying other layering metadata are -rejected. +# LiveKit Capture + +Video capture sources, and the machinery that publishes them with the LiveKit +[Rust SDK](../livekit/README.md). A capture backend implements one small trait. An application then +runs and supervises every backend the same way. + +## Source, pump, running pump + +Three concepts make up the crate. Video reaches a LiveKit track in one of two +forms, so the source and the pump each have two variants. + +**A source** produces frames or access units, one blocking call at a time. It +is the only trait a backend implements. A `pixel::PixelVideoSource` produces +libwebrtc `VideoFrame`s from a device such as a camera, and the WebRTC encoder +encodes them. An `encoded::EncodedVideoSource` produces access units from a +producer that encoded them already, such as an encoding pipeline. Passthrough +sends those to the wire with no re-encode. + +**A pump** bridges one source into a publishable RTC track: +`pixel::PixelVideoPump` or `encoded::EncodedVideoPump`. It builds the +matching RTC video source, derives the publish options, and runs the capture +loop. + +**A running pump** is a pump on a dedicated thread. Both pump kinds spawn into +the same `pump::RunningPump`, so an application supervises pumps of either kind +the same way. + +Sources block, so the pumps are synchronous code on plain threads. Only pump +construction needs the context of the async runtime that drives the SDK. + +## Publishing a track + +A pump supplies both pieces that the SDK needs, so publication is the same for +either path. + +```rust +let pump = PixelVideoPump::new(DemoSource::new(config)?); + +let track = LocalVideoTrack::create_video_track("demo", pump.rtc_source()); +let options = pump.publish_options(); +room.local_participant().publish_track(LocalTrack::Video(track), options).await?; + +let running = pump.spawn()?; +let stats = running.stop_and_join_async().await?; +``` + +## Sources + +Each source lives in its own module under `sources`, behind the Cargo feature +of the same name. Its module documents it. + +| Feature | Source | Path | +| ----------- | ---------------------- | ------- | +| `demo` | `DemoSource` | pixel | +| `gstreamer` | `GStreamerVideoSource` | encoded | From 54c8f20102cc6731327fd4da2f55819b311493be Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:52:23 -0700 Subject: [PATCH 35/56] Rename source for consistency --- livekit-capture/README.md | 4 ++-- livekit-capture/src/sources/demo.rs | 34 +++++++++++++-------------- livekit-ffi/src/conversion/capture.rs | 4 ++-- livekit-ffi/src/server/capture.rs | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 82f48f44c..357f46147 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -34,7 +34,7 @@ A pump supplies both pieces that the SDK needs, so publication is the same for either path. ```rust -let pump = PixelVideoPump::new(DemoSource::new(config)?); +let pump = PixelVideoPump::new(DemoVideoSource::new(config)?); let track = LocalVideoTrack::create_video_track("demo", pump.rtc_source()); let options = pump.publish_options(); @@ -51,5 +51,5 @@ of the same name. Its module documents it. | Feature | Source | Path | | ----------- | ---------------------- | ------- | -| `demo` | `DemoSource` | pixel | +| `demo` | `DemoVideoSource` | pixel | | `gstreamer` | `GStreamerVideoSource` | encoded | diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index c9c0934dd..ab8cdb8b5 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Solid-color demo source for testing. +//! Solid-color demo video source for testing. use crate::{error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop}; use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; @@ -35,7 +35,7 @@ const PALETTE: [(u8, u8, u8); 6] = [ (0x8E, 0x44, 0xAD), // purple ]; -/// Configuration for a [`DemoSource`]. +/// Configuration for a [`DemoVideoSource`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg_attr( feature = "serde", @@ -43,7 +43,7 @@ const PALETTE: [(u8, u8, u8); 6] = [ serde(deny_unknown_fields) )] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -pub struct DemoSourceConfig { +pub struct DemoVideoSourceConfig { /// Output resolution. pub resolution: VideoResolution, /// Output frame rate in frames per second. @@ -58,23 +58,23 @@ pub struct DemoSourceConfig { /// to validate capture integration end to end without a device or pipeline /// dependency. #[derive(Debug)] -pub struct DemoSource { - config: DemoSourceConfig, +pub struct DemoVideoSource { + config: DemoVideoSourceConfig, /// One `(y, u, v)` sample triple per palette color. colors: Vec<(u8, u8, u8)>, started: Option, frame_index: u64, } -impl DemoSource { +impl DemoVideoSource { /// Creates a demo source, rejecting an invalid configuration. - pub fn new(config: DemoSourceConfig) -> Result { + pub fn new(config: DemoVideoSourceConfig) -> Result { let VideoResolution { width, height } = config.resolution; if width == 0 || height == 0 { - return Err(SourceError::new(DemoSourceConfigError::ZeroResolution)); + return Err(SourceError::new(DemoVideoSourceConfigError::ZeroResolution)); } if config.framerate_fps == 0 { - return Err(SourceError::new(DemoSourceConfigError::ZeroFramerate)); + return Err(SourceError::new(DemoVideoSourceConfigError::ZeroFramerate)); } let colors = PALETTE.iter().map(|&color| yuv_from_rgb(color)).collect(); @@ -86,9 +86,9 @@ impl DemoSource { } } -/// Error returned when a [`DemoSourceConfig`] cannot produce frames. +/// Error returned when a [`DemoVideoSourceConfig`] cannot produce frames. #[derive(Debug, Error)] -pub enum DemoSourceConfigError { +pub enum DemoVideoSourceConfigError { /// The configured resolution has a zero component. #[error("demo source resolution must be non-zero")] ZeroResolution, @@ -97,7 +97,7 @@ pub enum DemoSourceConfigError { ZeroFramerate, } -impl PixelVideoSource for DemoSource { +impl PixelVideoSource for DemoVideoSource { fn resolution(&self) -> VideoResolution { self.config.resolution } @@ -149,8 +149,8 @@ fn yuv_from_rgb((r, g, b): (u8, u8, u8)) -> (u8, u8, u8) { mod tests { use super::*; - fn test_config() -> DemoSourceConfig { - DemoSourceConfig { + fn test_config() -> DemoVideoSourceConfig { + DemoVideoSourceConfig { resolution: VideoResolution { width: 64, height: 36 }, framerate_fps: 1000, } @@ -158,7 +158,7 @@ mod tests { #[test] fn yields_frames_with_configured_dimensions() { - let mut source = DemoSource::new(test_config()).unwrap(); + let mut source = DemoVideoSource::new(test_config()).unwrap(); let frame = source.next_frame(&PumpStop::new()).unwrap().unwrap(); assert_eq!((frame.buffer.width(), frame.buffer.height()), (64, 36)); @@ -172,7 +172,7 @@ mod tests { #[test] fn timestamps_follow_the_frame_rate() { - let mut source = DemoSource::new(test_config()).unwrap(); + let mut source = DemoVideoSource::new(test_config()).unwrap(); let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); let second = source.next_frame(&PumpStop::new()).unwrap().unwrap(); @@ -186,7 +186,7 @@ mod tests { // The source paces itself in real time, so this trades frame count // for the ~COLOR_INTERVAL the test spends sleeping either way. let frame_interval = COLOR_INTERVAL / 2; - let mut source = DemoSource::new(DemoSourceConfig { + let mut source = DemoVideoSource::new(DemoVideoSourceConfig { resolution: VideoResolution { width: 64, height: 36 }, framerate_fps: (Duration::from_secs(1).as_micros() / frame_interval.as_micros()) as u32, }) diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index 7672e7492..bc7d844c0 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -17,7 +17,7 @@ use livekit_capture::{ encoded::EncodedVideoCodec, primitive::VideoResolution, sources::{ - demo::DemoSourceConfig, + demo::DemoVideoSourceConfig, gstreamer::{GStreamerBitrateUnit, GStreamerRateControlConfig, GStreamerVideoSourceConfig}, }, }; @@ -28,7 +28,7 @@ impl From for VideoResolution { } } -impl From for DemoSourceConfig { +impl From for DemoVideoSourceConfig { fn from(config: proto::DemoVideoSourceConfig) -> Self { Self { resolution: config.resolution.into(), framerate_fps: config.framerate_fps } } diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index 97892bc83..58e6dbc5d 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -23,7 +23,7 @@ use livekit_capture::{ encoded::{EncodedVideoPump, EncodedVideoSource}, pixel::{PixelVideoPump, PixelVideoSource}, pump::{PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, - sources::{demo::DemoSource, gstreamer::GStreamerVideoSource}, + sources::{demo::DemoVideoSource, gstreamer::GStreamerVideoSource}, }; use parking_lot::Mutex; @@ -113,7 +113,7 @@ async fn create_capture_source( CapturePump::Encoded(EncodedVideoPump::new(source)) } proto::new_capture_source_request::Config::Demo(config) => { - let source = DemoSource::new(config.into()) + let source = DemoVideoSource::new(config.into()) .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; let source: Box = Box::new(source); CapturePump::Pixel(PixelVideoPump::new(source)) From 757a446ca2d94f5b6b3dc3db31d6bc1bbef62b30 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:59:24 -0700 Subject: [PATCH 36/56] Device facade --- Cargo.lock | 74 + livekit-capture/Cargo.toml | 57 +- livekit-capture/README.md | 1 + .../src/sources/device/avfoundation.rs | 1498 +++++++++++++++++ livekit-capture/src/sources/device/mod.rs | 624 +++++++ .../src/sources/device/timestamp.rs | 65 + livekit-capture/src/sources/device/v4l2.rs | 1150 +++++++++++++ livekit-capture/src/sources/gstreamer.rs | 6 + livekit-capture/src/sources/mod.rs | 3 + livekit-ffi/Cargo.toml | 4 +- livekit-ffi/protocol/capture.proto | 94 ++ livekit-ffi/protocol/ffi.proto | 9 +- livekit-ffi/src/conversion/capture.rs | 108 ++ livekit-ffi/src/server/capture.rs | 39 +- livekit-ffi/src/server/requests.rs | 7 +- 15 files changed, 3731 insertions(+), 8 deletions(-) create mode 100644 livekit-capture/src/sources/device/avfoundation.rs create mode 100644 livekit-capture/src/sources/device/mod.rs create mode 100644 livekit-capture/src/sources/device/timestamp.rs create mode 100644 livekit-capture/src/sources/device/v4l2.rs diff --git a/Cargo.lock b/Cargo.lock index 0ad050a63..bc5e50207 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1902,6 +1902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.13.1", + "block2 0.6.2", "objc2 0.6.4", ] @@ -4159,14 +4160,24 @@ name = "livekit-capture" version = "0.1.0" dependencies = [ "bytes", + "dispatch2", "gstreamer", "gstreamer-app", + "image", + "libc", "livekit", "log", + "objc2 0.6.4", + "objc2-av-foundation", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation 0.3.2", "schemars", "serde", "thiserror 2.0.19", "tokio", + "v4l", + "yuv-sys", ] [[package]] @@ -5061,6 +5072,19 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2 0.6.4", + "objc2-core-media", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -5096,6 +5120,28 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.1", + "objc2 0.6.4", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -5186,6 +5232,21 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + [[package]] name = "objc2-core-text" version = "0.3.2" @@ -5198,6 +5259,19 @@ dependencies = [ "objc2-core-graphics", ] +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.13.1", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", +] + [[package]] name = "objc2-encode" version = "4.1.0" diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index ee10daa9b..4c517af84 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -17,12 +17,13 @@ schemars = { workspace = true, optional = true } serde = { workspace = true, features = ["derive"], optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["sync"] } +yuv-sys = { workspace = true, features = ["jpeg"], optional = true } [dev-dependencies] tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] -default = ["demo", "gstreamer"] # TODO: Remove after testing +default = ["demo", "device", "gstreamer"] # TODO: Remove after testing serde = ["dep:serde"] schemars = ["dep:schemars", "serde"] @@ -32,6 +33,60 @@ tokio = ["tokio/rt"] # Pixel sources demo = [] +device = [ + "dep:yuv-sys", + # macOS backend + "dep:dispatch2", + "dep:objc2", + "dep:objc2-av-foundation", + "dep:objc2-core-media", + "dep:objc2-core-video", + "dep:objc2-foundation", + "objc2-av-foundation/AVCaptureDevice", + "objc2-av-foundation/AVCaptureInput", + "objc2-av-foundation/AVCaptureOutputBase", + "objc2-av-foundation/AVCaptureSession", + "objc2-av-foundation/AVCaptureSessionPreset", + "objc2-av-foundation/AVCaptureVideoDataOutput", + "objc2-av-foundation/AVMediaFormat", + "objc2-av-foundation/AVVideoSettings", + "objc2-av-foundation/dispatch2", + "objc2-av-foundation/objc2-core-media", + "objc2-core-media/CMFormatDescription", + "objc2-core-media/CMSync", + "objc2-core-media/CMTime", + "objc2-core-media/CMSampleBuffer", + "objc2-core-media/objc2-core-video", + "objc2-core-video/CVBase", + "objc2-core-video/CVBuffer", + "objc2-core-video/CVImageBuffer", + "objc2-core-video/CVPixelBuffer", + "objc2-core-video/CVReturn", + "objc2-foundation/NSArray", + "objc2-foundation/NSDictionary", + "objc2-foundation/NSError", + "objc2-foundation/NSObject", + "objc2-foundation/NSValue", + "objc2-foundation/NSString", + "objc2-foundation/objc2-core-foundation", + # Linux backend + "dep:image", + "dep:libc", + "dep:v4l", +] # Encoded sources gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] + +[target.'cfg(target_os = "macos")'.dependencies] +dispatch2 = { version = "0.3.1", default-features = false, features = ["std"], optional = true } +objc2 = { version = "0.6.4", default-features = false, features = ["std"], optional = true } +objc2-av-foundation = { version = "0.3.2", default-features = false, optional = true } +objc2-core-media = { version = "0.3.2", default-features = false, optional = true } +objc2-core-video = { version = "0.3.2", default-features = false, optional = true } +objc2-foundation = { version = "0.3.2", default-features = false, features = ["std"], optional = true } + +[target.'cfg(target_os = "linux")'.dependencies] +image = { workspace = true, optional = true } +libc = { version = "0.2", optional = true } +v4l = { version = "0.14", default-features = false, features = ["v4l2"], optional = true } diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 357f46147..51f77f81b 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -52,4 +52,5 @@ of the same name. Its module documents it. | Feature | Source | Path | | ----------- | ---------------------- | ------- | | `demo` | `DemoVideoSource` | pixel | +| `device` | `DeviceVideoSource` | pixel | | `gstreamer` | `GStreamerVideoSource` | encoded | diff --git a/livekit-capture/src/sources/device/avfoundation.rs b/livekit-capture/src/sources/device/avfoundation.rs new file mode 100644 index 000000000..e6d5ec457 --- /dev/null +++ b/livekit-capture/src/sources/device/avfoundation.rs @@ -0,0 +1,1498 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! macOS device capture backend built on AVFoundation. +//! +//! This module is an implementation detail of [`super::DeviceVideoSource`]; +//! nothing AVFoundation-specific leaves it. Frames are delivered as native +//! IOSurface-backed `CVPixelBuffer`s when the negotiated session supports it +//! (full-range NV12 without software scaling), and converted to I420 +//! otherwise. + +use std::ffi::c_void; +use std::ops::Deref; +use std::ptr::NonNull; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +use dispatch2::{DispatchQueue, DispatchRetained}; +use livekit::webrtc::video_frame::{ + native::NativeBuffer, BoxVideoFrame, I420Buffer, VideoBuffer, VideoFrame, VideoRotation, +}; +use objc2::rc::Retained; +use objc2::runtime::{AnyObject, ProtocolObject}; +use objc2::{define_class, msg_send, AnyThread, DefinedClass, Message}; +use objc2_av_foundation::{ + AVCaptureDevice, AVCaptureDeviceFormat, AVCaptureDeviceInput, AVCaptureOutput, + AVCaptureSession, AVCaptureSessionPreset1280x720, AVCaptureSessionPreset1920x1080, + AVCaptureSessionPreset640x480, AVCaptureSessionPresetHigh, AVCaptureSessionPresetInputPriority, + AVCaptureSessionPresetMedium, AVCaptureVideoDataOutput, + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureVideoStabilizationMode, + AVMediaTypeVideo, +}; +use objc2_core_media::{ + CMClock, CMSampleBuffer, CMTime, CMTimeFlags, CMVideoFormatDescriptionGetDimensions, +}; +use objc2_core_video::{ + kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferMetalCompatibilityKey, + kCVPixelBufferPixelFormatTypeKey, kCVPixelFormatType_32BGRA, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, + kCVPixelFormatType_420YpCbCr8PlanarFullRange, kCVPixelFormatType_422YpCbCr8, + kCVPixelFormatType_422YpCbCr8FullRange, kCVPixelFormatType_422YpCbCr8_yuvs, kCVReturnSuccess, + CVImageBuffer, CVPixelBuffer, CVPixelBufferGetBaseAddress, CVPixelBufferGetBaseAddressOfPlane, + CVPixelBufferGetBytesPerRow, CVPixelBufferGetBytesPerRowOfPlane, CVPixelBufferGetHeight, + CVPixelBufferGetHeightOfPlane, CVPixelBufferGetPixelFormatType, CVPixelBufferGetPlaneCount, + CVPixelBufferGetWidth, CVPixelBufferGetWidthOfPlane, CVPixelBufferLockBaseAddress, + CVPixelBufferLockFlags, CVPixelBufferUnlockBaseAddress, +}; +use objc2_foundation::{NSDictionary, NSNumber, NSObject, NSObjectProtocol, NSString}; + +use super::timestamp::{ + elapsed_us, unix_time_us_now, validate_capture_timestamp_us, MAX_CAPTURE_TIMESTAMP_AGE_US, +}; +use super::{ + capture_frame_metadata, DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, + DeviceSelector, DeviceVideoSourceConfig, DeviceVideoSourceError, +}; +use crate::{primitive::VideoResolution, pump::PumpStop}; + +unsafe extern "C" { + fn CFRelease(cf: *const c_void); + fn CVPixelBufferGetIOSurface(pixel_buffer: *const CVPixelBuffer) -> *const c_void; +} + +/// How long session construction waits for the device's first frame, which +/// establishes the delivered format. +const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long one frame wait may block before the stop token is rechecked. +const STOP_CHECK_INTERVAL: Duration = Duration::from_millis(100); + +/// Returns whether the backend can request this frame format from a device. +fn is_supported_request_format(frame_format: DeviceFrameFormat) -> bool { + matches!( + frame_format, + DeviceFrameFormat::Nv12 | DeviceFrameFormat::Bgra | DeviceFrameFormat::I420 + ) +} + +/// AVFoundation capture session satisfying the backend contract. +pub(super) struct Session { + format: DeviceFormat, + target_resolution: Option, + native_frame_supported: bool, + inner: SessionInner, +} + +// SAFETY: `Session` owns AVFoundation objects and only exposes `&mut self` +// frame capture plus `Drop`; moving ownership to another thread does not +// create concurrent access to those Objective-C objects. +unsafe impl Send for Session {} + +impl Session { + /// Opens a capture session and waits for the first frame to establish + /// the delivered format. + pub(super) fn open(config: &DeviceVideoSourceConfig) -> Result { + super::validate_config(config, is_supported_request_format)?; + + let inner = SessionInner::new(config)?; + let initial_frame = inner.wait_for_format(FIRST_FRAME_TIMEOUT)?; + inner.discard_pending_frame(); + let mut format = initial_frame.format; + format.framerate_fps = requested_framerate(&config.format).unwrap_or(30); + let target_resolution = requested_output_resolution(&config.format, format.resolution); + if let Some(resolution) = target_resolution { + format.resolution = resolution; + } + let session = Self { + format, + target_resolution, + native_frame_supported: initial_frame.native_frame_supported, + inner, + }; + log::info!( + "Opened device \"{}\" ({}): {} ({})", + session.inner.device_name, + session.inner.device_id, + session.format, + if session.native_capture() { "native buffers" } else { "converted to I420" }, + ); + Ok(session) + } + + /// Returns the negotiated capture format. + pub(super) fn format(&self) -> DeviceFormat { + self.format + } + + fn native_capture(&self) -> bool { + self.native_frame_supported + && self.target_resolution.is_none() + && self.format.frame_format == DeviceFrameFormat::Nv12 + } + + /// Blocks until the next frame is available, returning `Ok(None)` once + /// the stop token fires. + pub(super) fn next_frame( + &mut self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + // Convert only after the frame queue's mutex is released: conversion + // locks the pixel buffer and runs a full-frame libyuv copy, and + // holding the mutex through that would block `push_frame` on the + // AVFoundation delegate queue, which drops camera frames while + // stalled (`setAlwaysDiscardsLateVideoFrames(true)`). + let Some(queued) = self.inner.wait_take_queued_frame(stop)? else { + return Ok(None); + }; + + if self.native_capture() { + return queued.into_native_frame().map(|frame| Some(box_frame(frame))); + } + + let mut frame = queued.into_i420_frame()?; + if let Some(resolution) = self.target_resolution { + if frame.buffer.width() != resolution.width + || frame.buffer.height() != resolution.height + { + let width = i32::try_from(resolution.width).map_err(|_| { + DeviceVideoSourceError::InvalidFrame("scaled width exceeds i32") + })?; + let height = i32::try_from(resolution.height).map_err(|_| { + DeviceVideoSourceError::InvalidFrame("scaled height exceeds i32") + })?; + frame.buffer = frame.buffer.scale(width, height); + } + } + Ok(Some(box_frame(frame))) + } +} + +/// Type-erases a concrete frame for the pixel source contract. +fn box_frame + 'static>( + frame: VideoFrame, +) -> BoxVideoFrame { + VideoFrame { + rotation: frame.rotation, + timestamp_us: frame.timestamp_us, + frame_metadata: frame.frame_metadata, + buffer: Box::new(frame.buffer), + } +} + +/// Lists AVFoundation video capture devices. +pub(super) fn devices() -> Result, DeviceVideoSourceError> { + // SAFETY: AVMediaTypeVideo is a framework-provided immutable NSString + // constant. We only borrow it to ask AVFoundation for video devices. + let media_type = unsafe { AVMediaTypeVideo }.ok_or(DeviceVideoSourceError::DeviceNotFound)?; + // SAFETY: AVFoundation returns an immutable NSArray of currently available + // AVCaptureDevice instances. We only retain/copy string properties from it. + #[allow(deprecated)] + let devices = unsafe { AVCaptureDevice::devicesWithMediaType(media_type) }; + + let mut results = Vec::with_capacity(devices.len()); + for device in devices.iter() { + // SAFETY: These Objective-C property getters return retained NSStrings + // for a live AVCaptureDevice from the immutable devices array. + let id = unsafe { device.uniqueID() }.to_string(); + let name = unsafe { device.localizedName() }.to_string(); + let model_id = non_empty_string(unsafe { device.modelID() }.to_string()); + let manufacturer = non_empty_string(unsafe { device.manufacturer() }.to_string()); + + results.push(DeviceInfo { + id, + name, + model_id, + manufacturer, + formats: Vec::new(), + formats_complete: false, + }); + } + + Ok(results) +} + +fn non_empty_string(value: String) -> Option { + (!value.is_empty()).then_some(value) +} + +fn requested_output_resolution( + request: &DeviceFormatRequest, + delivered: VideoResolution, +) -> Option { + let DeviceFormatRequest::Closest(format) = request else { + return None; + }; + if format.resolution == delivered { + return None; + } + (resolution_area(format.resolution) <= resolution_area(delivered)).then_some(format.resolution) +} + +fn resolution_area(resolution: VideoResolution) -> u64 { + resolution.width as u64 * resolution.height as u64 +} + +struct SessionInner { + session: Retained, + _input: Retained, + output: Retained, + _delegate: Retained, + _queue: DispatchRetained, + shared: Arc, + device_name: String, + device_id: String, +} + +impl Drop for SessionInner { + fn drop(&mut self) { + self.shared.stop(); + // SAFETY: The output and session are owned by this wrapper. Clearing + // the delegate before stopping prevents callbacks from racing with + // the delegate being released during teardown. + unsafe { + self.output.setSampleBufferDelegate_queue(None, None); + self.session.stopRunning(); + } + } +} + +impl SessionInner { + fn new(config: &DeviceVideoSourceConfig) -> Result { + let device = select_device(&config.device)?; + // SAFETY: These property getters return retained NSStrings for a + // live AVCaptureDevice. + let device_name = unsafe { device.localizedName() }.to_string(); + let device_id = unsafe { device.uniqueID() }.to_string(); + let session = unsafe { AVCaptureSession::new() }; + let input = unsafe { AVCaptureDeviceInput::deviceInputWithDevice_error(&device) }.map_err( + |err| DeviceVideoSourceError::Backend(err.localizedDescription().to_string()), + )?; + let output = unsafe { AVCaptureVideoDataOutput::new() }; + let shared = Arc::new(FrameQueue::default()); + let delegate = CaptureDelegate::new(shared.clone()); + let queue = DispatchQueue::new("io.livekit.capture.device", None); + let active_format = select_active_format(&device, &config.format)?; + + // SAFETY: The session is newly created and not running. We add a + // camera input and video data output only after canAdd* checks. + unsafe { + session.beginConfiguration(); + session.setAutomaticallyConfiguresCaptureDeviceForWideColor(false); + if active_format.is_none() { + if let Some(preset) = session_preset(&config.format) { + session.setSessionPreset(preset); + } + } + let config_result = (|| { + if !session.canAddInput(&input) { + return Err(DeviceVideoSourceError::Backend( + "capture device input could not be added".to_string(), + )); + } + session.addInput(&input); + + configure_device(&device, &config.format, active_format.as_deref())?; + if active_format.is_some() + && session.canSetSessionPreset(AVCaptureSessionPresetInputPriority) + { + session.setSessionPreset(AVCaptureSessionPresetInputPriority); + } + configure_input_frame_duration(&input, &device, &config.format); + + if let Some(video_settings) = preferred_video_settings(&output) { + output.setVideoSettings(Some(&video_settings)); + } + output.setAlwaysDiscardsLateVideoFrames(true); + output.setSampleBufferDelegate_queue( + Some(ProtocolObject::from_ref(&*delegate)), + Some(&queue), + ); + if !session.canAddOutput(&output) { + return Err(DeviceVideoSourceError::Backend( + "video data output could not be added".to_string(), + )); + } + session.addOutput(&output); + configure_output_connection(&output)?; + Ok(()) + })(); + session.commitConfiguration(); + config_result?; + } + + // SAFETY: Configuration has been committed and the session is ready + // to synchronously start delivering video samples. + unsafe { + session.startRunning(); + } + + Ok(Self { + session, + _input: input, + output, + _delegate: delegate, + _queue: queue, + shared, + device_name, + device_id, + }) + } + + fn wait_for_format( + &self, + timeout: Duration, + ) -> Result { + self.shared.wait_for_format(timeout) + } + + fn wait_take_queued_frame( + &self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + self.shared.wait_take_queued_frame(stop) + } + + fn discard_pending_frame(&self) { + self.shared.discard_latest(); + } +} + +fn preferred_video_settings( + output: &AVCaptureVideoDataOutput, +) -> Option>> { + let preferred = [ + // WebRTC's VideoToolbox H.264 encoder allocates full-range NV12 + // buffers for its CPU upload path. Prefer the same CoreVideo + // format for direct CVPixelBuffer input so the native path does + // not have to reset VideoToolbox into a separate video-range pool. + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + ]; + // SAFETY: `output` is a live AVCaptureVideoDataOutput owned by the session setup path, and + // querying advertised CV pixel formats does not mutate Rust-managed memory. + let supported_formats = unsafe { output.availableVideoCVPixelFormatTypes() }; + let pixel_format_type = preferred + .into_iter() + .find(|preferred| supported_formats.iter().any(|format| format.as_u32() == *preferred))?; + + let pixel_format = NSNumber::new_u32(pixel_format_type); + let metal_compatible = NSNumber::new_bool(true); + let iosurface_properties = NSDictionary::::new(); + // SAFETY: The CoreVideo constants are immutable CFString keys. + // `CFString` and `NSString` are toll-free bridged, which + // objc2-foundation exposes through `AsRef`. + let pixel_format_key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); + // SAFETY: Same as above. + let iosurface_key: &NSString = unsafe { kCVPixelBufferIOSurfacePropertiesKey }.as_ref(); + // SAFETY: Same as above. + let metal_key: &NSString = unsafe { kCVPixelBufferMetalCompatibilityKey }.as_ref(); + Some(NSDictionary::from_slices( + &[pixel_format_key, iosurface_key, metal_key], + &[pixel_format.as_ref(), iosurface_properties.as_ref(), metal_compatible.as_ref()], + )) +} + +fn configure_input_frame_duration( + input: &AVCaptureDeviceInput, + device: &AVCaptureDevice, + request: &DeviceFormatRequest, +) { + let Some(framerate) = requested_framerate(request).filter(|framerate| *framerate > 0) else { + return; + }; + // SAFETY: `input` is the live input just added to the session. The + // support predicate is checked before setting the locked duration. + if !unsafe { input.isLockedVideoFrameDurationSupported() } { + return; + } + + let duration = unsafe { CMTime::with_seconds(1.0 / framerate as f64, 600) }; + // SAFETY: `device` and `input` belong to the same session setup path. + // The requested rate has already been checked against the active format + // before the device frame durations are set, and `input` reports locked + // frame duration support. + unsafe { + if device_format_supports_framerate(&device.activeFormat(), framerate) { + input.setActiveLockedVideoFrameDuration(duration); + } + } +} + +fn configure_output_connection( + output: &AVCaptureVideoDataOutput, +) -> Result<(), DeviceVideoSourceError> { + let media_type = unsafe { AVMediaTypeVideo }.ok_or(DeviceVideoSourceError::DeviceNotFound)?; + // SAFETY: `output` has just been added to a configured session. Querying + // its video connection does not mutate Rust-managed memory. + let Some(connection) = (unsafe { output.connectionWithMediaType(media_type) }) else { + return Err(DeviceVideoSourceError::Backend( + "video data output connection was not created".to_string(), + )); + }; + + // Keep frame-duration control on the device/input path. The deprecated + // output connection frame-duration setters can change whether macOS + // delivers IOSurface-backed pixel buffers. + // SAFETY: The connection is the video data output connection. Each + // setter is guarded by the corresponding support/configuration checks + // required by AVFoundation's API contract. + unsafe { + if connection.isVideoStabilizationSupported() { + connection.setPreferredVideoStabilizationMode(AVCaptureVideoStabilizationMode::Off); + } + if connection.automaticallyAdjustsVideoMirroring() { + connection.setAutomaticallyAdjustsVideoMirroring(false); + } + if connection.isVideoMirroringSupported() && connection.isVideoMirrored() { + connection.setVideoMirrored(false); + } + } + Ok(()) +} + +#[derive(Debug)] +struct CaptureDelegateIvars { + shared: Arc, +} + +define_class!( + // SAFETY: + // - The superclass NSObject does not have subclassing requirements. + // - CaptureDelegate does not implement Drop; retained Rust state lives in ivars. + #[unsafe(super = NSObject)] + #[thread_kind = AnyThread] + #[ivars = CaptureDelegateIvars] + struct CaptureDelegate; + + // SAFETY: `NSObjectProtocol` has no additional safety requirements. + unsafe impl NSObjectProtocol for CaptureDelegate {} + + // SAFETY: The selector signatures match the generated AVFoundation protocol. + unsafe impl AVCaptureVideoDataOutputSampleBufferDelegate for CaptureDelegate { + #[unsafe(method(captureOutput:didOutputSampleBuffer:fromConnection:))] + #[allow(non_snake_case)] + unsafe fn captureOutput_didOutputSampleBuffer_fromConnection( + &self, + _output: &AVCaptureOutput, + sample_buffer: &CMSampleBuffer, + _connection: &objc2_av_foundation::AVCaptureConnection, + ) { + if let Err(err) = process_sample_buffer(sample_buffer, &self.ivars().shared) { + self.ivars().shared.set_error(err.to_string()); + } + } + } +); + +impl CaptureDelegate { + fn new(shared: Arc) -> Retained { + let this = Self::alloc().set_ivars(CaptureDelegateIvars { shared }); + // SAFETY: `this` is freshly allocated and initialized exactly once + // using NSObject's designated initializer. + unsafe { msg_send![super(this), init] } + } +} + +/// Latest-frame mailbox shared between the AVFoundation delegate queue and +/// the capturing thread. +#[derive(Debug)] +struct FrameQueue { + state: Mutex, + ready: Condvar, + started_at: Instant, +} + +impl Default for FrameQueue { + fn default() -> Self { + Self { + state: Mutex::new(FrameQueueState::default()), + ready: Condvar::new(), + started_at: Instant::now(), + } + } +} + +#[derive(Debug, Default)] +struct FrameQueueState { + latest: Option, + stopped: bool, + error: Option, +} + +#[derive(Debug)] +struct InitialFrameInfo { + format: DeviceFormat, + native_frame_supported: bool, +} + +impl FrameQueue { + fn push_frame(&self, frame: QueuedFrame) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + if state.stopped { + return; + } + state.latest = Some(frame); + self.ready.notify_one(); + } + + fn set_error(&self, error: String) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + state.error = Some(error); + self.ready.notify_all(); + } + + /// Signals session teardown and wakes every blocked frame wait. + /// + /// Stopping is idempotent. `push_frame` discards frames delivered after + /// this point. + fn stop(&self) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + state.stopped = true; + self.ready.notify_all(); + } + + fn discard_latest(&self) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + state.latest = None; + } + + fn wait_for_format( + &self, + timeout: Duration, + ) -> Result { + let deadline = Instant::now() + timeout; + let mut state = self.state.lock().expect("device frame queue poisoned"); + loop { + if let Some(frame) = state.latest.as_ref() { + return Ok(InitialFrameInfo { + format: DeviceFormat::new( + VideoResolution::new(frame.width, frame.height), + 0, + frame.source_format, + ), + native_frame_supported: frame.native_frame_supported(), + }); + } + if let Some(error) = state.error.take() { + return Err(DeviceVideoSourceError::Backend(error)); + } + if state.stopped { + return Err(DeviceVideoSourceError::Backend( + "capture session stopped before delivering a frame".to_string(), + )); + } + + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(DeviceVideoSourceError::FrameTimeout); + }; + let (next_state, _) = + self.ready.wait_timeout(state, remaining).expect("device frame queue poisoned"); + state = next_state; + } + } + + /// Blocks until a frame, a delegate error, or a stop arrives and moves + /// the frame out of the shared state, returning `Ok(None)` on stop. + /// + /// The state mutex guard is dropped when this returns, so callers convert + /// the fully owned frame without holding the lock. Each wait is bounded + /// by [`STOP_CHECK_INTERVAL`] so the stop token is observed promptly even + /// when the device stalls without delivering frames or errors. + fn wait_take_queued_frame( + &self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + let mut state = self.state.lock().expect("device frame queue poisoned"); + loop { + if let Some(frame) = state.latest.take() { + return Ok(Some(frame)); + } + if let Some(error) = state.error.take() { + return Err(DeviceVideoSourceError::Backend(error)); + } + if state.stopped || stop.is_stopped() { + return Ok(None); + } + let (next_state, _) = self + .ready + .wait_timeout(state, STOP_CHECK_INTERVAL) + .expect("device frame queue poisoned"); + state = next_state; + } + } + + fn timestamp_us(&self) -> i64 { + elapsed_us(self.started_at.elapsed()) + } +} + +#[derive(Debug)] +struct QueuedFrame { + pixel_buffer: RetainedPixelBuffer, + width: u32, + height: u32, + source_format: DeviceFrameFormat, + core_video_pixel_format: u32, + /// Wall-clock capture time: the validated sensor timestamp when + /// AVFoundation reports one, the read time otherwise. + capture_wall_time_us: u64, + timestamp_us: i64, + is_iosurface_backed: bool, +} + +impl QueuedFrame { + fn into_i420_frame(self) -> Result, DeviceVideoSourceError> { + let buffer = convert_pixel_buffer(self.pixel_buffer.as_ref())?; + Ok(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: self.timestamp_us, + frame_metadata: Some(capture_frame_metadata(self.capture_wall_time_us)), + buffer, + }) + } + + fn into_native_frame(self) -> Result, DeviceVideoSourceError> { + if !self.native_frame_supported() { + return Err(DeviceVideoSourceError::Backend( + "native capture requires an IOSurface-backed full-range NV12 buffer".to_string(), + )); + } + + let timestamp_us = self.timestamp_us; + let capture_wall_time_us = self.capture_wall_time_us; + let buffer = self.pixel_buffer.into_native_buffer(); + Ok(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us, + frame_metadata: Some(capture_frame_metadata(capture_wall_time_us)), + buffer, + }) + } + + fn native_frame_supported(&self) -> bool { + self.source_format == DeviceFrameFormat::Nv12 + && self.core_video_pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + && self.is_iosurface_backed + } +} + +fn pixel_buffer_has_iosurface(pixel_buffer: &CVPixelBuffer) -> bool { + // SAFETY: `pixel_buffer` is a valid CVPixelBufferRef. CoreVideo returns + // an unretained IOSurfaceRef; this code only checks for null and does + // not store or release the returned pointer. + !unsafe { CVPixelBufferGetIOSurface(pixel_buffer) }.is_null() +} + +#[derive(Debug)] +struct RetainedPixelBuffer { + ptr: NonNull, +} + +// SAFETY: `RetainedPixelBuffer` owns a +1 CoreFoundation reference to a +// CVPixelBuffer. CoreFoundation retain/release and CoreVideo pixel-buffer +// inspection are thread-safe for this usage, and mutable pixel access still +// goes through CoreVideo's lock/unlock API. +unsafe impl Send for RetainedPixelBuffer {} +// SAFETY: The wrapper exposes only shared access to the pixel buffer and +// releases its retained reference on drop. +unsafe impl Sync for RetainedPixelBuffer {} + +impl RetainedPixelBuffer { + fn from_image_buffer(image_buffer: T) -> Self + where + T: Deref, + { + let ptr = NonNull::from(&*image_buffer).cast::(); + std::mem::forget(image_buffer); + Self { ptr } + } + + fn as_ref(&self) -> &CVPixelBuffer { + // SAFETY: `ptr` was created from a retained CVImageBuffer returned + // by CMSampleBufferGetImageBuffer and remains valid until this + // wrapper drops or transfers that retain. + unsafe { self.ptr.as_ref() } + } + + fn into_native_buffer(self) -> NativeBuffer { + let ptr = self.ptr.as_ptr().cast::(); + std::mem::forget(self); + // SAFETY: `ptr` is a valid retained CVPixelBufferRef. The WebRTC + // bridge wraps it in RTCCVPixelBuffer and then releases the +1 + // retain we transfer here, so Rust must not release it afterwards. + unsafe { NativeBuffer::from_cv_pixel_buffer(ptr) } + } +} + +impl Drop for RetainedPixelBuffer { + fn drop(&mut self) { + // SAFETY: `ptr` owns one CoreFoundation retain unless ownership was + // transferred by `into_native_buffer`, which forgets `self`. + unsafe { CFRelease(self.ptr.as_ptr().cast::()) }; + } +} + +fn select_device( + selector: &DeviceSelector, +) -> Result, DeviceVideoSourceError> { + let media_type = unsafe { AVMediaTypeVideo }.ok_or(DeviceVideoSourceError::DeviceNotFound)?; + match selector { + DeviceSelector::Default => { + unsafe { AVCaptureDevice::defaultDeviceWithMediaType(media_type) } + .ok_or(DeviceVideoSourceError::DeviceNotFound) + } + DeviceSelector::Index(index) => { + #[allow(deprecated)] + let devices = unsafe { AVCaptureDevice::devicesWithMediaType(media_type) }; + devices + .iter() + .nth(*index) + .map(|device| device.retain()) + .ok_or(DeviceVideoSourceError::DeviceNotFound) + } + DeviceSelector::Id(id) => { + let id = NSString::from_str(id); + unsafe { AVCaptureDevice::deviceWithUniqueID(&id) } + .ok_or(DeviceVideoSourceError::DeviceNotFound) + } + } +} + +fn select_active_format( + device: &AVCaptureDevice, + request: &DeviceFormatRequest, +) -> Result>, DeviceVideoSourceError> { + match request { + DeviceFormatRequest::Default => Ok(None), + DeviceFormatRequest::Exact(format) => { + let selected = best_device_format( + device, + Some(format.resolution), + Some(format.framerate_fps), + SelectionMode::Exact, + ); + selected.map(Some).ok_or(DeviceVideoSourceError::UnsupportedFormat(*format)) + } + DeviceFormatRequest::Closest(format) => Ok(best_device_format( + device, + Some(format.resolution), + Some(format.framerate_fps), + SelectionMode::Closest, + )), + DeviceFormatRequest::HighestFramerate { resolution, .. } => { + Ok(best_device_format(device, *resolution, None, SelectionMode::HighestFramerate)) + } + DeviceFormatRequest::HighestResolution { framerate_fps, .. } => { + Ok(best_device_format(device, None, *framerate_fps, SelectionMode::HighestResolution)) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SelectionMode { + Exact, + Closest, + HighestFramerate, + HighestResolution, +} + +#[derive(Debug)] +struct DeviceFormatCandidate { + format: Retained, + resolution: VideoResolution, + framerate_supported: bool, + max_framerate: u32, +} + +fn best_device_format( + device: &AVCaptureDevice, + resolution: Option, + framerate: Option, + mode: SelectionMode, +) -> Option> { + // SAFETY: The AVCaptureDevice is retained for the session setup path; querying the + // immutable list of supported formats does not mutate Rust-managed memory. + let formats = unsafe { device.formats() }; + let mut candidates = formats + .iter() + .filter_map(|format| { + let candidate_resolution = device_format_resolution(&format)?; + let framerate_supported = framerate + .map(|framerate| device_format_supports_framerate(&format, framerate)) + .unwrap_or(true); + Some(DeviceFormatCandidate { + format: format.retain(), + resolution: candidate_resolution, + framerate_supported, + max_framerate: device_format_max_framerate(&format), + }) + }) + .collect::>(); + + if let Some(resolution) = resolution { + if mode == SelectionMode::Exact { + return candidates + .into_iter() + .find(|candidate| { + candidate.resolution == resolution && candidate.framerate_supported + }) + .map(|candidate| candidate.format); + } + } + + if framerate.is_some() && candidates.iter().any(|candidate| candidate.framerate_supported) { + candidates.retain(|candidate| candidate.framerate_supported); + } + + match mode { + SelectionMode::Exact => None, + SelectionMode::Closest => { + let resolution = resolution?; + candidates + .into_iter() + .min_by_key(|candidate| resolution_distance(candidate.resolution, resolution)) + .map(|candidate| candidate.format) + } + SelectionMode::HighestFramerate => candidates + .into_iter() + .filter(|candidate| { + resolution.map(|resolution| candidate.resolution == resolution).unwrap_or(true) + }) + .max_by_key(|candidate| { + ( + candidate.max_framerate, + candidate.resolution.width as u64 * candidate.resolution.height as u64, + ) + }) + .map(|candidate| candidate.format), + SelectionMode::HighestResolution => candidates + .into_iter() + .max_by_key(|candidate| { + ( + candidate.resolution.width as u64 * candidate.resolution.height as u64, + candidate.max_framerate, + ) + }) + .map(|candidate| candidate.format), + } +} + +fn device_format_resolution(format: &AVCaptureDeviceFormat) -> Option { + // SAFETY: `format` is an AVCaptureDeviceFormat from the device's immutable formats array. + // Its format description is a valid CMVideoFormatDescription for video capture formats. + let description = unsafe { format.formatDescription() }; + // SAFETY: `description` is the video format description returned by AVFoundation. + let dimensions = unsafe { CMVideoFormatDescriptionGetDimensions(&description) }; + if dimensions.width <= 0 || dimensions.height <= 0 { + return None; + } + Some(VideoResolution::new(dimensions.width as u32, dimensions.height as u32)) +} + +fn device_format_supports_framerate(format: &AVCaptureDeviceFormat, framerate: u32) -> bool { + let requested = framerate as f64; + // SAFETY: `format` is an AVCaptureDeviceFormat from the device's immutable formats array. + // The returned frame-rate ranges are immutable AVFoundation objects. + unsafe { format.videoSupportedFrameRateRanges() }.iter().any(|range| { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + let min = unsafe { range.minFrameRate() }; + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + let max = unsafe { range.maxFrameRate() }; + requested >= min.floor() && requested <= max.ceil() + }) +} + +fn device_format_max_framerate(format: &AVCaptureDeviceFormat) -> u32 { + // SAFETY: `format` is an AVCaptureDeviceFormat from the device's immutable formats array. + // The returned frame-rate ranges are immutable AVFoundation objects. + unsafe { format.videoSupportedFrameRateRanges() } + .iter() + .map(|range| { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + unsafe { range.maxFrameRate() }.floor().max(0.0) as u32 + }) + .max() + .unwrap_or_default() +} + +fn resolution_distance(actual: VideoResolution, requested: VideoResolution) -> u64 { + let width_delta = actual.width.abs_diff(requested.width) as u64; + let height_delta = actual.height.abs_diff(requested.height) as u64; + let pixel_delta = (actual.width as u64 * actual.height as u64) + .abs_diff(requested.width as u64 * requested.height as u64); + pixel_delta + width_delta * width_delta + height_delta * height_delta +} + +fn configure_device( + device: &AVCaptureDevice, + request: &DeviceFormatRequest, + active_format: Option<&AVCaptureDeviceFormat>, +) -> Result<(), DeviceVideoSourceError> { + let framerate = requested_framerate(request); + if active_format.is_none() && framerate.is_none() { + return Ok(()); + } + + unsafe { device.lockForConfiguration() } + .map_err(|err| DeviceVideoSourceError::Backend(err.localizedDescription().to_string()))?; + + let configure_result = configure_locked_device(device, active_format, framerate); + // SAFETY: The device was successfully locked above and must be unlocked exactly once. + unsafe { + device.unlockForConfiguration(); + } + configure_result +} + +fn configure_locked_device( + device: &AVCaptureDevice, + active_format: Option<&AVCaptureDeviceFormat>, + framerate: Option, +) -> Result<(), DeviceVideoSourceError> { + // SAFETY: The caller holds the AVCaptureDevice configuration lock, and `active_format` + // was selected from this device's formats array. + unsafe { + if let Some(active_format) = active_format { + device.setActiveFormat(active_format); + } + } + configure_low_latency_device_processing(device); + + let Some(framerate) = framerate.filter(|framerate| *framerate > 0) else { + return Ok(()); + }; + + let active_format = match active_format { + Some(active_format) => active_format.retain(), + // SAFETY: The caller holds the configuration lock, and reading activeFormat is valid. + None => unsafe { device.activeFormat() }, + }; + if !device_format_supports_framerate(&active_format, framerate) { + return Ok(()); + } + + let duration = unsafe { CMTime::with_seconds(1.0 / framerate as f64, 600) }; + // SAFETY: The device is locked for configuration and the CMTime value is finite. + unsafe { + device.setActiveVideoMinFrameDuration(duration); + device.setActiveVideoMaxFrameDuration(duration); + } + Ok(()) +} + +fn configure_low_latency_device_processing(device: &AVCaptureDevice) { + // SAFETY: The caller holds the AVCaptureDevice configuration lock. + // Setters are guarded by their support/current-state predicates where + // AVFoundation requires that. + unsafe { + if device.automaticallyAdjustsVideoHDREnabled() { + device.setAutomaticallyAdjustsVideoHDREnabled(false); + } + if device.isVideoHDREnabled() { + device.setVideoHDREnabled(false); + } + if device.isLowLightBoostSupported() + && device.automaticallyEnablesLowLightBoostWhenAvailable() + { + device.setAutomaticallyEnablesLowLightBoostWhenAvailable(false); + } + if device.isSmoothAutoFocusSupported() && device.isSmoothAutoFocusEnabled() { + device.setSmoothAutoFocusEnabled(false); + } + } +} + +fn requested_framerate(request: &DeviceFormatRequest) -> Option { + match request { + DeviceFormatRequest::Default => None, + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + Some(format.framerate_fps) + } + DeviceFormatRequest::HighestFramerate { .. } => None, + DeviceFormatRequest::HighestResolution { framerate_fps, .. } => *framerate_fps, + } +} + +fn session_preset( + request: &DeviceFormatRequest, +) -> Option<&'static objc2_av_foundation::AVCaptureSessionPreset> { + let resolution = match request { + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + Some(format.resolution) + } + DeviceFormatRequest::HighestFramerate { resolution, .. } => *resolution, + DeviceFormatRequest::Default | DeviceFormatRequest::HighestResolution { .. } => None, + }?; + + exact_session_preset(resolution).or(Some(unsafe { AVCaptureSessionPresetHigh })) +} + +fn exact_session_preset( + resolution: VideoResolution, +) -> Option<&'static objc2_av_foundation::AVCaptureSessionPreset> { + match (resolution.width, resolution.height) { + (1920, 1080) => Some(unsafe { AVCaptureSessionPreset1920x1080 }), + (1280, 720) => Some(unsafe { AVCaptureSessionPreset1280x720 }), + (640, 480) => Some(unsafe { AVCaptureSessionPreset640x480 }), + (w, h) if w <= 640 && h <= 480 => Some(unsafe { AVCaptureSessionPresetMedium }), + _ => None, + } +} + +fn process_sample_buffer( + sample_buffer: &CMSampleBuffer, + shared: &FrameQueue, +) -> Result<(), DeviceVideoSourceError> { + let read_wall_time_us = unix_time_us_now().unwrap_or_default(); + let sensor_timestamp_us = sample_buffer_capture_wall_time_us(sample_buffer, read_wall_time_us); + let image_buffer = unsafe { sample_buffer.image_buffer() } + .ok_or(DeviceVideoSourceError::InvalidFrame("sample buffer has no image buffer"))?; + let pixel_buffer = RetainedPixelBuffer::from_image_buffer(image_buffer); + let pixel_buffer_ref = pixel_buffer.as_ref(); + let width = u32::try_from(CVPixelBufferGetWidth(pixel_buffer_ref)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("width is out of range"))?; + let height = u32::try_from(CVPixelBufferGetHeight(pixel_buffer_ref)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("height is out of range"))?; + let core_video_pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer_ref); + let source_format = frame_format_from_core_video(core_video_pixel_format)?; + let is_iosurface_backed = pixel_buffer_has_iosurface(pixel_buffer_ref); + + let capture_wall_time_us = sensor_timestamp_us.unwrap_or(read_wall_time_us); + shared.push_frame(QueuedFrame { + pixel_buffer, + width, + height, + source_format, + core_video_pixel_format, + capture_wall_time_us, + timestamp_us: shared.timestamp_us(), + is_iosurface_backed, + }); + Ok(()) +} + +fn sample_buffer_capture_wall_time_us( + sample_buffer: &CMSampleBuffer, + read_wall_time_us: u64, +) -> Option { + let sample_time = unsafe { sample_buffer.presentation_time_stamp() }; + + let timestamp_us = cm_time_to_us(sample_time)?; + if validate_capture_timestamp_us(timestamp_us, read_wall_time_us).is_some() { + return Some(timestamp_us); + } + + let host_now_us = current_host_time_us()?; + let age_us = host_now_us.checked_sub(timestamp_us)?; + if age_us > MAX_CAPTURE_TIMESTAMP_AGE_US { + return None; + } + read_wall_time_us.checked_sub(age_us) +} + +fn current_host_time_us() -> Option { + // SAFETY: The CoreMedia host time clock is a process-wide singleton and + // reading it does not mutate Rust-managed memory. + let host_clock = unsafe { CMClock::host_time_clock() }; + // SAFETY: `host_clock` is a valid retained CoreMedia clock. + let host_time = unsafe { host_clock.time() }; + cm_time_to_us(host_time) +} + +fn cm_time_to_us(time: CMTime) -> Option { + let flags = time.flags; + if !flags.contains(CMTimeFlags::Valid) || flags.intersects(CMTimeFlags::ImpliedValueFlagsMask) { + return None; + } + + // SAFETY: `time` is a valid CMTime value returned by CoreMedia. Invalid + // and indefinite values were filtered above. + let seconds = unsafe { time.seconds() }; + if !seconds.is_finite() || seconds < 0.0 { + return None; + } + + let micros = seconds * 1_000_000.0; + (micros <= u64::MAX as f64).then_some(micros.round() as u64) +} + +fn convert_pixel_buffer( + pixel_buffer: &CVPixelBuffer, +) -> Result { + let lock_flags = CVPixelBufferLockFlags::ReadOnly; + let lock_result = unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, lock_flags) }; + if lock_result != kCVReturnSuccess { + return Err(DeviceVideoSourceError::InvalidFrame("CVPixelBuffer lock failed")); + } + + let result = convert_locked_pixel_buffer(pixel_buffer); + + // SAFETY: The pixel buffer was locked above with the same flags. + let unlock_result = unsafe { CVPixelBufferUnlockBaseAddress(pixel_buffer, lock_flags) }; + if unlock_result != kCVReturnSuccess { + return Err(DeviceVideoSourceError::InvalidFrame("CVPixelBuffer unlock failed")); + } + + result +} + +fn convert_locked_pixel_buffer( + pixel_buffer: &CVPixelBuffer, +) -> Result { + let width = u32::try_from(CVPixelBufferGetWidth(pixel_buffer)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("width is out of range"))?; + let height = u32::try_from(CVPixelBufferGetHeight(pixel_buffer)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("height is out of range"))?; + let source_format = + frame_format_from_core_video(CVPixelBufferGetPixelFormatType(pixel_buffer))?; + + match source_format { + DeviceFrameFormat::Nv12 => convert_nv12(pixel_buffer, width, height), + DeviceFrameFormat::Bgra => convert_bgra(pixel_buffer, width, height), + DeviceFrameFormat::I420 => convert_i420(pixel_buffer, width, height), + DeviceFrameFormat::Uyvy => convert_uyvy(pixel_buffer, width, height), + DeviceFrameFormat::Yuyv => convert_yuy2(pixel_buffer, width, height), + other => Err(DeviceVideoSourceError::UnsupportedFrameFormat(other)), + } +} + +fn frame_format_from_core_video( + pixel_format: u32, +) -> Result { + match pixel_format { + format + if format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + || format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange => + { + Ok(DeviceFrameFormat::Nv12) + } + format if format == kCVPixelFormatType_32BGRA => Ok(DeviceFrameFormat::Bgra), + format + if format == kCVPixelFormatType_420YpCbCr8Planar + || format == kCVPixelFormatType_420YpCbCr8PlanarFullRange => + { + Ok(DeviceFrameFormat::I420) + } + format if format == kCVPixelFormatType_422YpCbCr8 => Ok(DeviceFrameFormat::Uyvy), + format + if format == kCVPixelFormatType_422YpCbCr8_yuvs + || format == kCVPixelFormatType_422YpCbCr8FullRange => + { + Ok(DeviceFrameFormat::Yuyv) + } + other => Err(DeviceVideoSourceError::Backend(format!( + "unsupported CoreVideo pixel format 0x{other:08x}" + ))), + } +} + +fn convert_nv12( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + if CVPixelBufferGetPlaneCount(pixel_buffer) < 2 { + return Err(DeviceVideoSourceError::InvalidFrame("NV12 buffer has fewer than two planes")); + } + + let y = plane(pixel_buffer, 0)?; + let uv = plane(pixel_buffer, 1)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slices cover the locked CVPixelBuffer planes for the duration of this + // call, and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_NV12ToI420( + y.data.as_ptr(), + y.stride as i32, + uv.data.as_ptr(), + uv.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("NV12ToI420 failed")); + } + Ok(buffer) +} + +fn convert_bgra( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + let bgra = packed_plane(pixel_buffer, 4)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers the locked CVPixelBuffer for the duration of this call, + // and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_BGRAToI420( + bgra.data.as_ptr(), + bgra.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("BGRAToI420 failed")); + } + Ok(buffer) +} + +fn convert_uyvy( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + let uyvy = packed_plane(pixel_buffer, 2)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers the locked CVPixelBuffer for the duration of this call, + // and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_UYVYToI420( + uyvy.data.as_ptr(), + uyvy.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("UYVYToI420 failed")); + } + Ok(buffer) +} + +fn convert_yuy2( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + let yuy2 = packed_plane(pixel_buffer, 2)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers the locked CVPixelBuffer for the duration of this call, + // and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_YUY2ToI420( + yuy2.data.as_ptr(), + yuy2.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("YUY2ToI420 failed")); + } + Ok(buffer) +} + +fn convert_i420( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + if CVPixelBufferGetPlaneCount(pixel_buffer) < 3 { + return Err(DeviceVideoSourceError::InvalidFrame( + "I420 buffer has fewer than three planes", + )); + } + + let y = plane(pixel_buffer, 0)?; + let u = plane(pixel_buffer, 1)?; + let v = plane(pixel_buffer, 2)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slices cover the locked CVPixelBuffer planes for the duration of this + // call, and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_I420Copy( + y.data.as_ptr(), + y.stride as i32, + u.data.as_ptr(), + u.stride as i32, + v.data.as_ptr(), + v.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("I420Copy failed")); + } + Ok(buffer) +} + +struct Plane<'a> { + data: &'a [u8], + stride: usize, +} + +fn plane(pixel_buffer: &CVPixelBuffer, index: usize) -> Result, DeviceVideoSourceError> { + let plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + if index >= plane_count { + return Err(DeviceVideoSourceError::InvalidFrame("plane index is out of range")); + } + + let base = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, index); + if base.is_null() { + return Err(DeviceVideoSourceError::InvalidFrame("pixel plane has no base address")); + } + let stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, index); + let height = CVPixelBufferGetHeightOfPlane(pixel_buffer, index); + let width = CVPixelBufferGetWidthOfPlane(pixel_buffer, index); + let min_len = stride + .checked_mul(height.saturating_sub(1)) + .and_then(|value| value.checked_add(width)) + .ok_or(DeviceVideoSourceError::InvalidFrame("pixel plane size overflow"))?; + + // SAFETY: The CVPixelBuffer is locked for read-only access, the plane + // base address is non-null, and CoreVideo reports the minimum readable + // extent for this plane. + let data = unsafe { std::slice::from_raw_parts(base.cast::(), min_len) }; + Ok(Plane { data, stride }) +} + +fn packed_plane( + pixel_buffer: &CVPixelBuffer, + bytes_per_pixel: usize, +) -> Result, DeviceVideoSourceError> { + let base = CVPixelBufferGetBaseAddress(pixel_buffer); + if base.is_null() { + return Err(DeviceVideoSourceError::InvalidFrame("pixel buffer has no base address")); + } + let stride = CVPixelBufferGetBytesPerRow(pixel_buffer); + let height = CVPixelBufferGetHeight(pixel_buffer); + let width = CVPixelBufferGetWidth(pixel_buffer) + .checked_mul(bytes_per_pixel) + .ok_or(DeviceVideoSourceError::InvalidFrame("packed pixel row size overflow"))?; + let min_len = stride + .checked_mul(height.saturating_sub(1)) + .and_then(|value| value.checked_add(width)) + .ok_or(DeviceVideoSourceError::InvalidFrame("packed pixel buffer size overflow"))?; + + // SAFETY: The CVPixelBuffer is locked for read-only access, the base + // address is non-null, and CoreVideo reports the minimum readable extent + // for this packed buffer. + let data = unsafe { std::slice::from_raw_parts(base.cast::(), min_len) }; + Ok(Plane { data, stride }) +} + +#[cfg(test)] +mod tests { + use std::sync::{mpsc, Arc}; + use std::time::Duration; + + use super::{FrameQueue, STOP_CHECK_INTERVAL}; + use crate::pump::PumpStop; + + /// Upper bound on how long a woken frame wait may take to return before + /// the test declares the stop path broken. + const STOP_WAIT_TIMEOUT: Duration = Duration::from_secs(5); + + // `FrameQueue` is pure Rust state, so these tests run on macOS CI hosts + // without camera hardware or AVFoundation involvement. + + #[test] + fn stop_token_unblocks_frame_wait() { + let queue = Arc::new(FrameQueue::default()); + let stop = PumpStop::new(); + + let (done_tx, done_rx) = mpsc::channel(); + let stop_in_waiter = stop.clone(); + let queue_in_waiter = Arc::clone(&queue); + let waiter = std::thread::spawn(move || { + let result = queue_in_waiter.wait_take_queued_frame(&stop_in_waiter); + let _ = done_tx.send(()); + result + }); + + // Give the waiter time to block. There is no race if the stop lands + // first: the wait loop re-checks the token at least every + // STOP_CHECK_INTERVAL. + std::thread::sleep(Duration::from_millis(50)); + stop.stop(); + + done_rx + .recv_timeout(STOP_WAIT_TIMEOUT + STOP_CHECK_INTERVAL) + .expect("frame wait did not return after the stop token fired"); + let result = waiter.join().expect("frame wait thread panicked"); + assert!(matches!(result, Ok(None)), "unexpected frame wait result: {result:?}"); + } + + #[test] + fn frame_waits_return_none_once_queue_stopped() { + let queue = FrameQueue::default(); + queue.stop(); + // Stopping is idempotent. + queue.stop(); + + assert!(matches!(queue.wait_take_queued_frame(&PumpStop::new()), Ok(None))); + } + + #[test] + fn delegate_errors_surface_from_frame_wait() { + let queue = FrameQueue::default(); + queue.set_error("camera unplugged".to_string()); + + let error = queue + .wait_take_queued_frame(&PumpStop::new()) + .expect_err("delegate error must surface"); + assert!(error.to_string().contains("camera unplugged")); + } +} diff --git a/livekit-capture/src/sources/device/mod.rs b/livekit-capture/src/sources/device/mod.rs new file mode 100644 index 000000000..645a53e54 --- /dev/null +++ b/livekit-capture/src/sources/device/mod.rs @@ -0,0 +1,624 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Camera device capture. +//! +//! [`DeviceVideoSource`] is a pixel source that captures frames from a video +//! device attached to the machine, using the platform's native capture stack. +//! The platform integration is an implementation detail: configuration, +//! enumeration ([`devices`]), and errors share one platform-neutral +//! vocabulary, and the same configuration works on every supported platform. +//! On platforms without a capture backend the module still compiles; +//! construction and enumeration fail with +//! [`DeviceVideoSourceError::UnsupportedPlatform`]. +//! +//! Where the platform supports it, frames reach the RTC track as +//! platform-native buffers without a CPU copy; otherwise they are converted +//! to I420. + +#[cfg(target_os = "macos")] +mod avfoundation; +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod timestamp; +#[cfg(target_os = "linux")] +mod v4l2; + +#[cfg(target_os = "macos")] +use avfoundation as backend; +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +use unsupported as backend; +#[cfg(target_os = "linux")] +use v4l2 as backend; + +use std::fmt; + +use livekit::webrtc::video_frame::BoxVideoFrame; +use thiserror::Error; + +use crate::{ + error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop, +}; + +/// Selects the video device a [`DeviceVideoSource`] captures from. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "lowercase") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum DeviceSelector { + /// The platform default video device. + #[default] + Default, + /// The device at this position in the platform enumeration order. + Index(usize), + /// A platform-stable device identifier, as reported by [`DeviceInfo::id`]. + Id(String), +} + +/// Frame format delivered by a capture device. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "lowercase") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum DeviceFrameFormat { + /// Planar I420/YUV420P. + I420, + /// Biplanar NV12. + Nv12, + /// Packed BGRA. + Bgra, + /// Packed RGB24. + Rgb24, + /// Packed BGR24. + Bgr24, + /// Packed YUYV/YUY2. + Yuyv, + /// Packed UYVY. + Uyvy, + /// Single-plane 8-bit luma. + Grey, + /// Encoded MJPEG frames. + Mjpeg, +} + +impl DeviceFrameFormat { + /// Returns a stable lower-case frame-format name. + pub const fn as_str(self) -> &'static str { + match self { + Self::I420 => "i420", + Self::Nv12 => "nv12", + Self::Bgra => "bgra", + Self::Rgb24 => "rgb24", + Self::Bgr24 => "bgr24", + Self::Yuyv => "yuyv", + Self::Uyvy => "uyvy", + Self::Grey => "grey", + Self::Mjpeg => "mjpeg", + } + } +} + +impl fmt::Display for DeviceFrameFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for DeviceFrameFormat { + type Err = DeviceFrameFormatParseError; + + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "i420" | "yuv420p" => Ok(Self::I420), + "nv12" => Ok(Self::Nv12), + "bgra" => Ok(Self::Bgra), + "rgb24" | "rgb" => Ok(Self::Rgb24), + "bgr24" | "bgr" => Ok(Self::Bgr24), + "yuyv" | "yuy2" => Ok(Self::Yuyv), + "uyvy" => Ok(Self::Uyvy), + "grey" | "greyscale" => Ok(Self::Grey), + "mjpeg" | "mjpg" => Ok(Self::Mjpeg), + _ => Err(DeviceFrameFormatParseError), + } + } +} + +/// Error returned when parsing a [`DeviceFrameFormat`] from a string. +#[derive(Debug, Clone, Copy, Error, PartialEq, Eq)] +#[error("unknown device frame format")] +pub struct DeviceFrameFormatParseError; + +/// Capture format offered by a device. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeviceFormat { + /// Frame dimensions. + pub resolution: VideoResolution, + /// Frame rate in frames per second. + pub framerate_fps: u32, + /// Frame format. + pub frame_format: DeviceFrameFormat, +} + +impl DeviceFormat { + /// Creates a device capture format. + pub const fn new( + resolution: VideoResolution, + framerate_fps: u32, + frame_format: DeviceFrameFormat, + ) -> Self { + Self { resolution, framerate_fps, frame_format } + } +} + +impl fmt::Display for DeviceFormat { + /// Formats as `WIDTHxHEIGHT@FPSfps FORMAT`. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}@{}fps {}", self.resolution, self.framerate_fps, self.frame_format) + } +} + +/// Format selection requested from a capture device. +/// +/// The device negotiates the delivered format; [`DeviceVideoSource::format`] +/// reports the outcome. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "snake_case") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum DeviceFormatRequest { + /// Let the device choose its default format. + #[default] + Default, + /// Require an exact format match. + Exact(DeviceFormat), + /// Use the device's closest supported format. + Closest(DeviceFormat), + /// Prefer the highest frame rate, optionally constrained by resolution + /// and frame format. + HighestFramerate { + /// Optional resolution constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + resolution: Option, + /// Optional frame format constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + frame_format: Option, + }, + /// Prefer the highest resolution, optionally constrained by frame rate + /// and frame format. + HighestResolution { + /// Optional frame-rate constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + framerate_fps: Option, + /// Optional frame format constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + frame_format: Option, + }, +} + +/// Video capture device discovered by [`devices`]. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeviceInfo { + /// Platform-stable device identifier. + pub id: String, + /// Human-readable device name. + pub name: String, + /// Device model identifier, when available. + pub model_id: Option, + /// Device manufacturer, when available. + pub manufacturer: Option, + /// Capture formats reported by the device. + pub formats: Vec, + /// Whether [`DeviceInfo::formats`] is a complete list; some platforms do + /// not enumerate formats up front. + pub formats_complete: bool, +} + +impl DeviceInfo { + /// Returns the selector that reopens this exact device. + pub fn selector(&self) -> DeviceSelector { + DeviceSelector::Id(self.id.clone()) + } +} + +/// Lists the video capture devices available on this machine, running the +/// blocking enumeration on the tokio blocking pool. +/// +/// Requires a running tokio runtime; [`devices_blocking`] is the +/// non-async form. +#[cfg(feature = "tokio")] +pub async fn devices() -> Result, SourceError> { + match tokio::task::spawn_blocking(devices_blocking).await { + Ok(result) => result, + Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), + Err(err) => Err(SourceError::new(err)), + } +} + +/// Lists the video capture devices available on this machine. +/// +/// Enumeration queries the platform capture stack and may block briefly. +pub fn devices_blocking() -> Result, SourceError> { + backend::devices().map_err(SourceError::new) +} + +/// Configuration for a [`DeviceVideoSource`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeviceVideoSourceConfig { + /// Device to capture from. + #[cfg_attr(feature = "serde", serde(default))] + pub device: DeviceSelector, + /// Format requested from the device. + #[cfg_attr(feature = "serde", serde(default))] + pub format: DeviceFormatRequest, +} + +/// Pixel video source that captures frames from a video device such as a +/// camera. +/// +/// Construction opens the device and negotiates the capture format, so +/// [`DeviceVideoSource::format`] and the source's nominal resolution are +/// known before the first frame is pumped. Devices never reach end of +/// stream; stop the pump driving the source instead. +/// +/// Frames carry a monotonic `timestamp_us`, and each frame's +/// `frame_metadata` is pre-filled with the wall-clock capture time (the +/// device's own capture timestamp when the platform reports a valid one). +pub struct DeviceVideoSource { + config: DeviceVideoSourceConfig, + format: DeviceFormat, + session: backend::Session, +} + +impl DeviceVideoSource { + /// Creates the source, running blocking device negotiation on the tokio + /// blocking pool. + /// + /// Requires a running tokio runtime. This is the async-constructor + /// convention for capture backends: `new` for async consumers, and + /// [`DeviceVideoSource::new_blocking`] for everything else. + #[cfg(feature = "tokio")] + pub async fn new(config: DeviceVideoSourceConfig) -> Result { + match tokio::task::spawn_blocking(move || Self::new_blocking(config)).await { + Ok(result) => result, + Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), + Err(err) => Err(SourceError::new(err)), + } + } + + /// Opens the configured device and negotiates the capture format. + /// + /// This blocks until the device delivers enough information to establish + /// the format — on some platforms that includes waiting for the first + /// frame, bounded by a timeout. Construction fails loudly on a missing + /// device, an unsatisfiable format request, or a platform without a + /// capture backend. + pub fn new_blocking(config: DeviceVideoSourceConfig) -> Result { + let session = backend::Session::open(&config).map_err(SourceError::new)?; + let format = session.format(); + Ok(Self { config, format, session }) + } + + /// Returns the configuration the source was created with. + pub fn config(&self) -> &DeviceVideoSourceConfig { + &self.config + } + + /// Returns the negotiated capture format. + /// + /// The resolution matches what [`PixelVideoSource::resolution`] reports; + /// the frame format is what the device delivers before any conversion. + pub fn format(&self) -> DeviceFormat { + self.format + } +} + +impl PixelVideoSource for DeviceVideoSource { + fn resolution(&self) -> VideoResolution { + self.format.resolution + } + + // Backends bound every blocking wait so the stop token is observed + // within ~100ms even when the device stalls. + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + self.session.next_frame(stop).map_err(SourceError::new) + } +} + +impl fmt::Debug for DeviceVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DeviceVideoSource") + .field("config", &self.config) + .field("format", &self.format) + .finish_non_exhaustive() + } +} + +/// Error returned by device capture. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum DeviceVideoSourceError { + /// Device capture has no backend for this platform. + #[error("device capture is not supported on this platform")] + UnsupportedPlatform, + /// The requested device was not found. + #[error("capture device was not found")] + DeviceNotFound, + /// The configuration is invalid. + #[error("invalid device source configuration: {0}")] + InvalidConfig(&'static str), + /// The requested frame format is not supported by this platform's + /// backend. + #[error("device capture does not support frame format {0} on this platform")] + UnsupportedFrameFormat(DeviceFrameFormat), + /// The requested capture format is not available on the selected device. + #[error("capture format is not available on the device: {0}")] + UnsupportedFormat(DeviceFormat), + /// Timed out waiting for the device to deliver a frame. + #[error("timed out waiting for a frame from the capture device")] + FrameTimeout, + /// Captured frame bytes did not match the negotiated format. + #[error("invalid captured frame: {0}")] + InvalidFrame(&'static str), + /// Pixel conversion failed. + #[error("failed to convert captured frame to I420: {0}")] + Convert(&'static str), + /// Compressed frame decoding failed. + #[error("failed to decode compressed frame: {0}")] + Decode(String), + /// The platform capture stack reported an error. + #[error("capture device error: {0}")] + Backend(String), +} + +/// Builds the packet-trailer metadata that device frames are pre-filled +/// with; a metadata callback set on the pump takes precedence. +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn capture_frame_metadata( + capture_wall_time_us: u64, +) -> livekit::webrtc::video_frame::FrameMetadata { + livekit::webrtc::video_frame::FrameMetadata { + user_timestamp: Some(capture_wall_time_us), + frame_id: None, + user_data: None, + } +} + +/// Validates the platform-neutral parts of a configuration; `supported` +/// reports whether the backend can deliver a frame format. +#[cfg_attr(not(any(target_os = "macos", target_os = "linux")), allow(dead_code))] +fn validate_config( + config: &DeviceVideoSourceConfig, + supported: fn(DeviceFrameFormat) -> bool, +) -> Result<(), DeviceVideoSourceError> { + if let DeviceSelector::Id(id) = &config.device { + if id.is_empty() { + return Err(DeviceVideoSourceError::InvalidConfig("device id must be non-empty")); + } + } + + let validate_frame_format = |frame_format: DeviceFrameFormat| { + if !supported(frame_format) { + return Err(DeviceVideoSourceError::UnsupportedFrameFormat(frame_format)); + } + Ok(()) + }; + let validate_resolution = |resolution: VideoResolution| { + if resolution.width == 0 { + return Err(DeviceVideoSourceError::InvalidConfig("width must be non-zero")); + } + if resolution.height == 0 { + return Err(DeviceVideoSourceError::InvalidConfig("height must be non-zero")); + } + Ok(()) + }; + + match &config.format { + DeviceFormatRequest::Default => Ok(()), + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + validate_resolution(format.resolution)?; + if format.framerate_fps == 0 { + return Err(DeviceVideoSourceError::InvalidConfig( + "framerate_fps must be non-zero", + )); + } + validate_frame_format(format.frame_format) + } + DeviceFormatRequest::HighestFramerate { resolution, frame_format } => { + if let Some(resolution) = resolution { + validate_resolution(*resolution)?; + } + if let Some(frame_format) = frame_format { + validate_frame_format(*frame_format)?; + } + Ok(()) + } + DeviceFormatRequest::HighestResolution { framerate_fps, frame_format } => { + if matches!(framerate_fps, Some(0)) { + return Err(DeviceVideoSourceError::InvalidConfig( + "framerate_fps must be non-zero", + )); + } + if let Some(frame_format) = frame_format { + validate_frame_format(*frame_format)?; + } + Ok(()) + } + } +} + +/// Stub backend for platforms without device capture. +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +mod unsupported { + use livekit::webrtc::video_frame::BoxVideoFrame; + + use super::{DeviceFormat, DeviceInfo, DeviceVideoSourceConfig, DeviceVideoSourceError}; + use crate::pump::PumpStop; + + /// Uninhabited: [`Session::open`] always fails on this platform. + #[derive(Debug)] + pub(super) enum Session {} + + impl Session { + pub(super) fn open( + _config: &DeviceVideoSourceConfig, + ) -> Result { + Err(DeviceVideoSourceError::UnsupportedPlatform) + } + + pub(super) fn format(&self) -> DeviceFormat { + match *self {} + } + + pub(super) fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + match *self {} + } + } + + pub(super) fn devices() -> Result, DeviceVideoSourceError> { + Err(DeviceVideoSourceError::UnsupportedPlatform) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + + fn any_supported(_: DeviceFrameFormat) -> bool { + true + } + + #[test] + fn frame_format_parses_common_names() { + assert_eq!(DeviceFrameFormat::from_str("MJPEG"), Ok(DeviceFrameFormat::Mjpeg)); + assert_eq!(DeviceFrameFormat::from_str("mjpg"), Ok(DeviceFrameFormat::Mjpeg)); + assert_eq!(DeviceFrameFormat::from_str("grey"), Ok(DeviceFrameFormat::Grey)); + assert_eq!(DeviceFrameFormat::from_str("GREY"), Ok(DeviceFrameFormat::Grey)); + assert_eq!(DeviceFrameFormat::from_str("yuy2"), Ok(DeviceFrameFormat::Yuyv)); + } + + #[test] + fn frame_format_displays_canonical_names() { + assert_eq!(DeviceFrameFormat::Mjpeg.to_string(), "mjpeg"); + assert_eq!(DeviceFrameFormat::Grey.to_string(), "grey"); + } + + #[test] + fn validation_rejects_empty_device_id() { + let config = DeviceVideoSourceConfig { + device: DeviceSelector::Id(String::new()), + format: DeviceFormatRequest::Default, + }; + assert!(matches!( + validate_config(&config, any_supported), + Err(DeviceVideoSourceError::InvalidConfig(_)) + )); + } + + #[test] + fn validation_rejects_zero_format_components() { + let zero_width = DeviceVideoSourceConfig { + device: DeviceSelector::Default, + format: DeviceFormatRequest::Exact(DeviceFormat::new( + VideoResolution::new(0, 720), + 30, + DeviceFrameFormat::Yuyv, + )), + }; + assert!(matches!( + validate_config(&zero_width, any_supported), + Err(DeviceVideoSourceError::InvalidConfig(_)) + )); + + let zero_framerate = DeviceVideoSourceConfig { + device: DeviceSelector::Default, + format: DeviceFormatRequest::HighestResolution { + framerate_fps: Some(0), + frame_format: None, + }, + }; + assert!(matches!( + validate_config(&zero_framerate, any_supported), + Err(DeviceVideoSourceError::InvalidConfig(_)) + )); + } + + #[test] + fn validation_rejects_unsupported_frame_formats() { + let config = DeviceVideoSourceConfig { + device: DeviceSelector::Default, + format: DeviceFormatRequest::HighestFramerate { + resolution: None, + frame_format: Some(DeviceFrameFormat::Uyvy), + }, + }; + assert!(matches!( + validate_config(&config, |format| format != DeviceFrameFormat::Uyvy), + Err(DeviceVideoSourceError::UnsupportedFrameFormat(DeviceFrameFormat::Uyvy)) + )); + } + + #[test] + fn default_config_requests_default_device_and_format() { + let config = DeviceVideoSourceConfig::default(); + assert_eq!(config.device, DeviceSelector::Default); + assert_eq!(config.format, DeviceFormatRequest::Default); + } + + #[test] + fn device_info_selector_reopens_by_id() { + let info = DeviceInfo { + id: "camera-0".to_string(), + name: "Camera".to_string(), + model_id: None, + manufacturer: None, + formats: Vec::new(), + formats_complete: false, + }; + assert_eq!(info.selector(), DeviceSelector::Id("camera-0".to_string())); + } +} diff --git a/livekit-capture/src/sources/device/timestamp.rs b/livekit-capture/src/sources/device/timestamp.rs new file mode 100644 index 000000000..eb842b435 --- /dev/null +++ b/livekit-capture/src/sources/device/timestamp.rs @@ -0,0 +1,65 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Capture-timestamp helpers shared by the device capture backends. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Maximum age a backend-reported capture timestamp may have, relative to the +/// wall-clock read time, before it is considered stale and discarded. +pub(super) const MAX_CAPTURE_TIMESTAMP_AGE_US: u64 = 5_000_000; + +/// Returns the current UNIX wall-clock time in microseconds. +pub(super) fn unix_time_us_now() -> Option { + let elapsed = SystemTime::now().duration_since(UNIX_EPOCH).ok()?; + u64::try_from(elapsed.as_micros()).ok() +} + +/// Converts a duration to whole microseconds, saturating at `i64::MAX`. +pub(super) fn elapsed_us(duration: Duration) -> i64 { + i64::try_from(duration.as_micros()).unwrap_or(i64::MAX) +} + +/// Validates a backend-reported capture timestamp against the wall-clock read +/// time: zero, future, and stale (older than +/// [`MAX_CAPTURE_TIMESTAMP_AGE_US`]) timestamps are rejected. +pub(super) fn validate_capture_timestamp_us( + capture_timestamp_us: u64, + read_wall_time_us: u64, +) -> Option { + if capture_timestamp_us == 0 || capture_timestamp_us > read_wall_time_us { + return None; + } + if read_wall_time_us - capture_timestamp_us > MAX_CAPTURE_TIMESTAMP_AGE_US { + return None; + } + Some(capture_timestamp_us) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validate_rejects_zero_future_and_stale_timestamps() { + let now = 10_000_000; + assert_eq!(validate_capture_timestamp_us(0, now), None); + assert_eq!(validate_capture_timestamp_us(now + 1, now), None); + assert_eq!( + validate_capture_timestamp_us(now - MAX_CAPTURE_TIMESTAMP_AGE_US - 1, now), + None + ); + assert_eq!(validate_capture_timestamp_us(now - 1, now), Some(now - 1)); + } +} diff --git a/livekit-capture/src/sources/device/v4l2.rs b/livekit-capture/src/sources/device/v4l2.rs new file mode 100644 index 000000000..f6518c9c9 --- /dev/null +++ b/livekit-capture/src/sources/device/v4l2.rs @@ -0,0 +1,1150 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Linux device capture backend built on V4L2. +//! +//! This module is an implementation detail of [`super::DeviceVideoSource`]; +//! nothing V4L2-specific leaves it. Frames are converted to I420 on the CPU +//! (via libyuv, with an image-crate fallback for MJPEG streams libyuv +//! rejects). + +use std::io; +use std::path::Path; +use std::time::{Duration, Instant}; + +use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; +use v4l::{ + buffer::{Flags as V4lBufferFlags, Type as V4lBufferType}, + capability::Flags as V4lCapabilityFlags, + context, + format::{Format as V4lFormat, FourCC}, + frameinterval::FrameIntervalEnum, + framesize::FrameSizeEnum, + io::{mmap::Stream as MmapStream, traits::CaptureStream}, + video::{capture::Parameters as V4lCaptureParameters, Capture}, + Device, +}; + +use super::timestamp::{elapsed_us, unix_time_us_now, validate_capture_timestamp_us}; +use super::{ + capture_frame_metadata, DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, + DeviceSelector, DeviceVideoSourceConfig, DeviceVideoSourceError, +}; +use crate::{primitive::VideoResolution, pump::PumpStop}; + +/// How long the stream's own wait may block. Only the first frame read (which +/// starts the stream) can hit this; later reads are gated on a poll and never +/// wait inside the stream. A stream wait that times out cannot be retried, so +/// a timeout here is a hard [`DeviceVideoSourceError::FrameTimeout`]. +const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long one fd poll may block before the stop token is rechecked, in +/// milliseconds. +const STOP_CHECK_INTERVAL_MS: i32 = 100; + +/// Number of memory-mapped buffers shared with the driver. +const BUFFER_COUNT: u32 = 4; + +/// Returns whether the backend can convert this source frame format. +fn is_supported_source_format(frame_format: DeviceFrameFormat) -> bool { + matches!( + frame_format, + DeviceFrameFormat::Nv12 + | DeviceFrameFormat::Rgb24 + | DeviceFrameFormat::Bgr24 + | DeviceFrameFormat::Yuyv + | DeviceFrameFormat::Grey + | DeviceFrameFormat::Mjpeg + ) +} + +/// Default ordered source frame formats to try, most preferred first. +fn default_frame_formats() -> Vec { + vec![ + DeviceFrameFormat::Yuyv, + DeviceFrameFormat::Mjpeg, + DeviceFrameFormat::Grey, + DeviceFrameFormat::Rgb24, + DeviceFrameFormat::Nv12, + ] +} + +/// V4L2 capture session satisfying the backend contract. +pub(super) struct Session { + device: Device, + stream: MmapStream<'static>, + format: DeviceFormat, + /// Driver-reported row stride in bytes (V4L2 `bytesperline`). + stride: u32, + started_at: Instant, + /// Frame pulled while starting the stream, handed out first. + pending_frame: Option, +} + +impl Session { + /// Opens the device, negotiates the capture format, and starts the + /// stream by pulling its first frame. + pub(super) fn open(config: &DeviceVideoSourceConfig) -> Result { + super::validate_config(config, is_supported_source_format)?; + + let frame_formats = frame_formats_for_request(&config.format); + let device = open_device(&config.device)?; + let device_name = device + .query_caps() + .ok() + .map(|caps| caps.card) + .filter(|card| !card.is_empty()) + .unwrap_or_else(|| "unknown".to_string()); + let all_formats = enumerate_device_formats(&device)?; + let (format, stride) = + apply_format_request(&device, &config.format, &frame_formats, &all_formats)?; + let mut stream = + MmapStream::with_buffers(&device, V4lBufferType::VideoCapture, BUFFER_COUNT) + .map_err(backend_error)?; + stream.set_timeout(FIRST_FRAME_TIMEOUT); + + let mut session = Self { + device, + stream, + format, + stride, + started_at: Instant::now(), + pending_frame: None, + }; + // Pull the first frame during construction: it queues the stream's + // buffers and starts streaming, so every later wait can be + // poll-bounded to observe the stop token, and it proves the + // negotiated format actually delivers frames. + let first_frame = session.read_frame()?; + session.pending_frame = Some(first_frame); + log::info!( + "Opened device \"{}\": {} (converted to I420)", + device_name, + session.format, + ); + Ok(session) + } + + /// Returns the negotiated capture format. + pub(super) fn format(&self) -> DeviceFormat { + self.format + } + + /// Blocks until the next frame is available, returning `Ok(None)` once + /// the stop token fires. + pub(super) fn next_frame( + &mut self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + if let Some(frame) = self.pending_frame.take() { + return Ok(Some(frame)); + } + + // Bounded fd polls keep the stop token observed within + // STOP_CHECK_INTERVAL_MS even when the device stalls. The stream is + // only read once the fd signals, so the stream's own wait — which + // cannot be resumed after a timeout — never blocks here. + loop { + if stop.is_stopped() { + return Ok(None); + } + match self.device.handle().poll(libc::POLLIN, STOP_CHECK_INTERVAL_MS) { + Ok(0) => continue, + // Readable, or an error condition the stream read surfaces. + Ok(_) => break, + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, + Err(err) => return Err(backend_error(err)), + } + } + self.read_frame().map(Some) + } + + /// Dequeues one frame from the stream and converts it to I420. + fn read_frame(&mut self) -> Result { + let fallback_wall_time_us = unix_time_us_now().unwrap_or_default(); + let format = self.format; + let stride = self.stride; + let (buffer, metadata) = self.stream.next().map_err(|err| match err.kind() { + io::ErrorKind::TimedOut => DeviceVideoSourceError::FrameTimeout, + _ => backend_error(err), + })?; + let timestamp_us = elapsed_us(self.started_at.elapsed()); + let read_wall_time_us = unix_time_us_now().unwrap_or(fallback_wall_time_us); + let backend_capture_timestamp = + v4l_timestamp_to_wallclock(metadata.timestamp, v4l_timestamp_clock(metadata.flags)); + let capture_wall_time_us = select_capture_wall_time_us( + backend_capture_timestamp, + fallback_wall_time_us, + read_wall_time_us, + ); + + let width = format.resolution.width; + let height = format.resolution.height; + let mut i420 = I420Buffer::new(width, height); + let source = frame_bytes(buffer, metadata.bytesused); + convert_to_i420(format.frame_format, source, width, height, stride, &mut i420)?; + + Ok(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us, + frame_metadata: Some(capture_frame_metadata(capture_wall_time_us)), + buffer: Box::new(i420), + }) + } +} + +/// Lists Linux V4L2 capture devices. +pub(super) fn devices() -> Result, DeviceVideoSourceError> { + let devices = context::enum_devices() + .into_iter() + .filter_map(|node| { + let id = node.index().to_string(); + let fallback_name = + node.name().unwrap_or_else(|| node.path().to_string_lossy().into_owned()); + let mut name = fallback_name; + let mut model_id = None; + let mut manufacturer = None; + let mut formats = Vec::new(); + let mut formats_complete = false; + + if let Ok(device) = Device::with_path(node.path()) { + if let Ok(capabilities) = device.query_caps() { + if !capabilities.capabilities.contains(V4lCapabilityFlags::VIDEO_CAPTURE) { + return None; + } + if !capabilities.card.is_empty() { + name = capabilities.card; + } + model_id = Some(capabilities.bus).filter(|value| !value.is_empty()); + manufacturer = Some(capabilities.driver).filter(|value| !value.is_empty()); + } + + if let Ok(device_formats) = enumerate_device_formats(&device) { + formats = device_formats; + formats_complete = true; + } + } + + Some(DeviceInfo { id, name, model_id, manufacturer, formats, formats_complete }) + }) + .collect(); + + Ok(devices) +} + +fn open_device(selector: &DeviceSelector) -> Result { + match selector { + DeviceSelector::Default => Device::new(0).map_err(open_error), + DeviceSelector::Index(index) => Device::new(*index).map_err(open_error), + DeviceSelector::Id(id) => open_device_id(id), + } +} + +fn open_device_id(id: &str) -> Result { + if let Ok(index) = id.parse::() { + return Device::new(index).map_err(open_error); + } + + Device::with_path(Path::new(id)).map_err(open_error) +} + +fn open_error(error: io::Error) -> DeviceVideoSourceError { + if error.kind() == io::ErrorKind::NotFound { + DeviceVideoSourceError::DeviceNotFound + } else { + backend_error(error) + } +} + +fn backend_error(error: io::Error) -> DeviceVideoSourceError { + DeviceVideoSourceError::Backend(error.to_string()) +} + +/// Returns the ordered source frame formats to try for a request. +/// +/// The request's own frame format (already validated as supported) is tried +/// first; an explicit constraint on the highest-* requests pins the list to +/// that one format. +fn frame_formats_for_request(request: &DeviceFormatRequest) -> Vec { + let mut formats = match request { + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + ordered_formats_with_first(&default_frame_formats(), format.frame_format) + } + DeviceFormatRequest::HighestFramerate { frame_format: Some(frame_format), .. } + | DeviceFormatRequest::HighestResolution { frame_format: Some(frame_format), .. } => { + vec![*frame_format] + } + DeviceFormatRequest::Default + | DeviceFormatRequest::HighestFramerate { frame_format: None, .. } + | DeviceFormatRequest::HighestResolution { frame_format: None, .. } => { + default_frame_formats() + } + }; + formats.dedup(); + formats +} + +fn ordered_formats_with_first( + frame_formats: &[DeviceFrameFormat], + first: DeviceFrameFormat, +) -> Vec { + std::iter::once(first) + .chain(frame_formats.iter().copied().filter(|format| *format != first)) + .collect() +} + +fn apply_format_request( + device: &Device, + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + match request { + DeviceFormatRequest::Default + | DeviceFormatRequest::HighestFramerate { .. } + | DeviceFormatRequest::HighestResolution { .. } => { + let selected = select_format_for_request(request, frame_formats, all_formats)?; + set_device_format(device, selected) + } + DeviceFormatRequest::Exact(_) | DeviceFormatRequest::Closest(_) => { + apply_ordered_format_request(device, request, frame_formats, all_formats) + } + } +} + +/// Tries the request once per candidate source frame format, in preference +/// order, returning the first format the device accepts. +fn apply_ordered_format_request( + device: &Device, + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + let mut last_error = None; + for frame_format in frame_formats { + let request = format_request_with_frame_format(request, *frame_format); + let selected = match select_format_for_request(&request, &[*frame_format], all_formats) { + Ok(selected) => selected, + Err(error) => { + last_error = Some(error); + continue; + } + }; + + match set_device_format(device, selected) { + Ok(format) => return Ok(format), + Err(error) => last_error = Some(error), + } + } + + Err(last_error + .unwrap_or(DeviceVideoSourceError::InvalidConfig("no source frame formats to request"))) +} + +fn format_request_with_frame_format( + request: &DeviceFormatRequest, + frame_format: DeviceFrameFormat, +) -> DeviceFormatRequest { + match request { + DeviceFormatRequest::Exact(format) => DeviceFormatRequest::Exact(DeviceFormat::new( + format.resolution, + format.framerate_fps, + frame_format, + )), + DeviceFormatRequest::Closest(format) => DeviceFormatRequest::Closest(DeviceFormat::new( + format.resolution, + format.framerate_fps, + frame_format, + )), + DeviceFormatRequest::Default => DeviceFormatRequest::Default, + DeviceFormatRequest::HighestFramerate { resolution, .. } => { + DeviceFormatRequest::HighestFramerate { + resolution: *resolution, + frame_format: Some(frame_format), + } + } + DeviceFormatRequest::HighestResolution { framerate_fps, .. } => { + DeviceFormatRequest::HighestResolution { + framerate_fps: *framerate_fps, + frame_format: Some(frame_format), + } + } + } +} + +fn select_format_for_request( + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Result { + let selected = match request { + DeviceFormatRequest::Default => { + all_formats.iter().find(|format| frame_formats.contains(&format.frame_format)).copied() + } + DeviceFormatRequest::Exact(format) => { + if frame_formats.contains(&format.frame_format) { + Some(*format) + } else { + None + } + } + DeviceFormatRequest::Closest(format) => { + select_closest_format(*format, frame_formats, all_formats) + } + DeviceFormatRequest::HighestFramerate { .. } => { + select_highest_framerate_format(request, frame_formats, all_formats) + } + DeviceFormatRequest::HighestResolution { .. } => { + select_highest_resolution_format(request, frame_formats, all_formats) + } + }; + + selected.ok_or_else(|| match request { + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + DeviceVideoSourceError::UnsupportedFormat(*format) + } + _ => DeviceVideoSourceError::Backend( + "no device format satisfies the format request".to_string(), + ), + }) +} + +fn select_closest_format( + requested: DeviceFormat, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Option { + if !frame_formats.contains(&requested.frame_format) { + return None; + } + + let resolution = all_formats + .iter() + .copied() + .filter(|format| format.frame_format == requested.frame_format) + .min_by_key(|format| resolution_distance(format.resolution, requested.resolution))? + .resolution; + + let framerate_fps = all_formats + .iter() + .copied() + .filter(|format| { + format.frame_format == requested.frame_format && format.resolution == resolution + }) + .min_by_key(|format| format.framerate_fps.abs_diff(requested.framerate_fps))? + .framerate_fps; + + Some(DeviceFormat::new(resolution, framerate_fps, requested.frame_format)) +} + +fn select_highest_framerate_format( + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Option { + all_formats + .iter() + .copied() + .filter(|format| frame_formats.contains(&format.frame_format)) + .filter(|format| match request { + DeviceFormatRequest::HighestFramerate { resolution, frame_format } => { + resolution.map(|resolution| format.resolution == resolution).unwrap_or(true) + && frame_format + .map(|frame_format| format.frame_format == frame_format) + .unwrap_or(true) + } + _ => false, + }) + .max_by(|left, right| { + left.framerate_fps + .cmp(&right.framerate_fps) + .then_with(|| compare_resolution(left.resolution, right.resolution)) + .then_with(|| { + compare_format_preference(left.frame_format, right.frame_format, frame_formats) + }) + }) +} + +fn select_highest_resolution_format( + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Option { + all_formats + .iter() + .copied() + .filter(|format| frame_formats.contains(&format.frame_format)) + .filter(|format| match request { + DeviceFormatRequest::HighestResolution { framerate_fps, frame_format } => { + framerate_fps + .map(|framerate_fps| format.framerate_fps == framerate_fps) + .unwrap_or(true) + && frame_format + .map(|frame_format| format.frame_format == frame_format) + .unwrap_or(true) + } + _ => false, + }) + .max_by(|left, right| { + compare_resolution(left.resolution, right.resolution) + .then_with(|| left.framerate_fps.cmp(&right.framerate_fps)) + .then_with(|| { + compare_format_preference(left.frame_format, right.frame_format, frame_formats) + }) + }) +} + +fn compare_resolution(left: VideoResolution, right: VideoResolution) -> std::cmp::Ordering { + frame_area(left) + .cmp(&frame_area(right)) + .then_with(|| left.width.cmp(&right.width)) + .then_with(|| left.height.cmp(&right.height)) +} + +fn resolution_distance(left: VideoResolution, right: VideoResolution) -> u64 { + let width = i64::from(left.width) - i64::from(right.width); + let height = i64::from(left.height) - i64::from(right.height); + width.unsigned_abs().pow(2) + height.unsigned_abs().pow(2) +} + +fn frame_area(resolution: VideoResolution) -> u64 { + u64::from(resolution.width) * u64::from(resolution.height) +} + +fn compare_format_preference( + left: DeviceFrameFormat, + right: DeviceFrameFormat, + frame_formats: &[DeviceFrameFormat], +) -> std::cmp::Ordering { + let left_index = frame_formats.iter().position(|format| *format == left).unwrap_or(usize::MAX); + let right_index = + frame_formats.iter().position(|format| *format == right).unwrap_or(usize::MAX); + right_index.cmp(&left_index) +} + +fn set_device_format( + device: &Device, + selected: DeviceFormat, +) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + let (current, _) = device_capture_format(device)?; + let format_changed = + current.resolution != selected.resolution || current.frame_format != selected.frame_format; + if format_changed { + device + .set_format(&V4lFormat::new( + selected.resolution.width, + selected.resolution.height, + fourcc_for_frame_format(selected.frame_format) + .ok_or(DeviceVideoSourceError::UnsupportedFrameFormat(selected.frame_format))?, + )) + .map_err(backend_error)?; + } + if format_changed || current.framerate_fps != selected.framerate_fps { + device + .set_params(&V4lCaptureParameters::with_fps(selected.framerate_fps)) + .map_err(backend_error)?; + } + + let (actual, stride) = device_capture_format(device)?; + if actual != selected { + return Err(DeviceVideoSourceError::Backend(format!( + "device rejected capture format: requested {selected}, got {actual}" + ))); + } + Ok((actual, stride)) +} + +/// Returns the device's current capture format and its row stride in bytes +/// (V4L2 `bytesperline`). +fn device_capture_format(device: &Device) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + let format = device.format().map_err(backend_error)?; + let params = device.params().map_err(backend_error)?; + let framerate_fps = + framerate_from_fraction(params.interval.numerator, params.interval.denominator).ok_or( + DeviceVideoSourceError::Backend("device reports a zero frame interval".to_string()), + )?; + let capture_format = DeviceFormat::new( + VideoResolution::new(format.width, format.height), + framerate_fps, + frame_format_from_fourcc(format.fourcc).ok_or_else(|| { + DeviceVideoSourceError::Backend(format!("unsupported fourcc {}", format.fourcc)) + })?, + ); + Ok((capture_format, format.stride)) +} + +fn enumerate_device_formats(device: &Device) -> Result, DeviceVideoSourceError> { + let mut formats = Vec::new(); + let fourccs = device + .enum_formats() + .map_err(backend_error)? + .into_iter() + .filter_map(|format| frame_format_from_fourcc(format.fourcc).map(|_| format.fourcc)) + .collect::>(); + + for fourcc in dedup_fourccs(fourccs) { + let Some(frame_format) = frame_format_from_fourcc(fourcc) else { + continue; + }; + let frame_sizes = device.enum_framesizes(fourcc).map_err(backend_error)?; + for resolution in frame_sizes.into_iter().flat_map(resolutions_from_frame_size) { + let intervals = device + .enum_frameintervals(fourcc, resolution.width, resolution.height) + .unwrap_or_default(); + for framerate_fps in intervals.into_iter().flat_map(framerates_from_interval) { + formats.push(DeviceFormat::new(resolution, framerate_fps, frame_format)); + } + } + } + + Ok(formats) +} + +fn fourcc_for_frame_format(frame_format: DeviceFrameFormat) -> Option { + match frame_format { + DeviceFrameFormat::Nv12 => Some(FourCC::new(b"NV12")), + DeviceFrameFormat::Rgb24 => Some(FourCC::new(b"RGB3")), + DeviceFrameFormat::Bgr24 => Some(FourCC::new(b"BGR3")), + DeviceFrameFormat::Yuyv => Some(FourCC::new(b"YUYV")), + DeviceFrameFormat::Grey => Some(FourCC::new(b"GREY")), + DeviceFrameFormat::Mjpeg => Some(FourCC::new(b"MJPG")), + DeviceFrameFormat::I420 | DeviceFrameFormat::Bgra | DeviceFrameFormat::Uyvy => None, + } +} + +fn frame_format_from_fourcc(fourcc: FourCC) -> Option { + match fourcc.str().ok()? { + "NV12" => Some(DeviceFrameFormat::Nv12), + "RGB3" => Some(DeviceFrameFormat::Rgb24), + "BGR3" => Some(DeviceFrameFormat::Bgr24), + "YUYV" | "YUY2" => Some(DeviceFrameFormat::Yuyv), + "GREY" => Some(DeviceFrameFormat::Grey), + "MJPG" | "JPEG" => Some(DeviceFrameFormat::Mjpeg), + _ => None, + } +} + +fn dedup_fourccs(fourccs: Vec) -> Vec { + let mut deduped = Vec::new(); + for fourcc in fourccs { + if !deduped.contains(&fourcc) { + deduped.push(fourcc); + } + } + deduped +} + +fn resolutions_from_frame_size(size: v4l::FrameSize) -> Vec { + match size.size { + FrameSizeEnum::Discrete(discrete) => { + vec![VideoResolution::new(discrete.width, discrete.height)] + } + FrameSizeEnum::Stepwise(stepwise) => { + let mut resolutions = Vec::new(); + push_stepwise_resolution( + &mut resolutions, + VideoResolution::new(stepwise.min_width, stepwise.min_height), + ); + push_stepwise_resolution( + &mut resolutions, + VideoResolution::new(stepwise.max_width, stepwise.max_height), + ); + resolutions + } + } +} + +fn push_stepwise_resolution(resolutions: &mut Vec, resolution: VideoResolution) { + if resolution.width != 0 && resolution.height != 0 && !resolutions.contains(&resolution) { + resolutions.push(resolution); + } +} + +fn framerates_from_interval(interval: v4l::FrameInterval) -> Vec { + match interval.interval { + FrameIntervalEnum::Discrete(fraction) => { + framerate_from_fraction(fraction.numerator, fraction.denominator).into_iter().collect() + } + FrameIntervalEnum::Stepwise(stepwise) => { + let mut framerates = Vec::new(); + for fraction in [stepwise.min, stepwise.max] { + if let Some(framerate) = + framerate_from_fraction(fraction.numerator, fraction.denominator) + { + if !framerates.contains(&framerate) { + framerates.push(framerate); + } + } + } + framerates + } + } +} + +/// Converts a V4L2 frame interval (seconds per frame) to frames per second. +/// +/// Non-integer rates (e.g. the NTSC interval 1001/30000 = 29.97fps) round to +/// the nearest whole rate, never below 1. +fn framerate_from_fraction(numerator: u32, denominator: u32) -> Option { + if numerator == 0 || denominator == 0 { + return None; + } + if denominator % numerator == 0 { + return Some(denominator / numerator); + } + let rounded = (u64::from(denominator) + u64::from(numerator) / 2) / u64::from(numerator); + Some(u32::try_from(rounded).unwrap_or(u32::MAX).max(1)) +} + +fn frame_bytes(buffer: &[u8], bytes_used: u32) -> &[u8] { + let bytes_used = usize::try_from(bytes_used).unwrap_or(buffer.len()).min(buffer.len()); + if bytes_used == 0 { + buffer + } else { + &buffer[..bytes_used] + } +} + +fn convert_to_i420( + source_format: DeviceFrameFormat, + source: &[u8], + width: u32, + height: u32, + source_stride: u32, + destination: &mut I420Buffer, +) -> Result<(), DeviceVideoSourceError> { + let (stride_y, stride_u, stride_v) = destination.strides(); + let (dst_y, dst_u, dst_v) = destination.data_mut(); + let width_i32 = i32_from_u32(width, "width exceeds supported range")?; + let height_i32 = i32_from_u32(height, "height exceeds supported range")?; + + let ret = match source_format { + DeviceFrameFormat::Yuyv => { + let stride = source_row_stride(source_stride, width as usize * 2); + validate_len(source, stride * height as usize, "YUYV frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_YUY2ToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Rgb24 => { + let stride = source_row_stride(source_stride, width as usize * 3); + validate_len(source, stride * height as usize, "RGB24 frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_RGB24ToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Bgr24 => { + let stride = source_row_stride(source_stride, width as usize * 3); + validate_len(source, stride * height as usize, "BGR24 frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_RAWToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Grey => { + let stride = source_row_stride(source_stride, width as usize); + validate_len(source, stride * height as usize, "GREY frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_I400ToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Nv12 => { + // Single-planar V4L2 NV12: the interleaved chroma plane follows the + // luma plane at `stride * height` and shares the luma stride. + let stride = source_row_stride(source_stride, width as usize); + let y_size = stride * height as usize; + validate_len(source, y_size + y_size / 2, "NV12 frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_NV12ToI420( + source.as_ptr(), + stride_i32, + source[y_size..].as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Mjpeg => { + return convert_mjpeg_to_i420(source, width, height, destination); + } + DeviceFrameFormat::I420 | DeviceFrameFormat::Bgra | DeviceFrameFormat::Uyvy => { + return Err(DeviceVideoSourceError::UnsupportedFrameFormat(source_format)); + } + }; + + if ret == 0 { + Ok(()) + } else { + Err(DeviceVideoSourceError::Convert("libyuv conversion failed")) + } +} + +/// Returns the effective source row stride in bytes, falling back to the +/// packed width-derived stride when the driver reports `bytesperline` as zero +/// or smaller than one packed row. +fn source_row_stride(reported_stride: u32, packed_stride: usize) -> usize { + (reported_stride as usize).max(packed_stride) +} + +fn convert_mjpeg_to_i420( + source: &[u8], + width: u32, + height: u32, + destination: &mut I420Buffer, +) -> Result<(), DeviceVideoSourceError> { + let (stride_y, stride_u, stride_v) = destination.strides(); + let (dst_y, dst_u, dst_v) = destination.data_mut(); + let width_i32 = i32_from_u32(width, "width exceeds supported range")?; + let height_i32 = i32_from_u32(height, "height exceeds supported range")?; + + // SAFETY: Source and destination slices are valid for the dimensions and strides. + let ret = unsafe { + yuv_sys::rs_MJPGToI420( + source.as_ptr(), + source.len(), + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + width_i32, + height_i32, + ) + }; + if ret == 0 { + return Ok(()); + } + + let rgb = image::load_from_memory(source) + .map_err(|error| DeviceVideoSourceError::Decode(error.to_string()))? + .to_rgb8(); + if rgb.width() != width || rgb.height() != height { + return Err(DeviceVideoSourceError::InvalidFrame("decoded MJPEG dimensions changed")); + } + // SAFETY: Source and destination slices are valid for the dimensions and strides. + let ret = unsafe { + yuv_sys::rs_RGB24ToI420( + rgb.as_raw().as_ptr(), + width_i32 * 3, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + }; + if ret == 0 { + Ok(()) + } else { + Err(DeviceVideoSourceError::Convert("RGB24 fallback conversion failed")) + } +} + +fn validate_len( + source: &[u8], + expected: usize, + label: &'static str, +) -> Result<(), DeviceVideoSourceError> { + if source.len() < expected { + return Err(DeviceVideoSourceError::InvalidFrame(label)); + } + Ok(()) +} + +/// Selects the wall-clock capture time for a frame: the validated +/// driver-reported timestamp when there is one, the read time otherwise. +fn select_capture_wall_time_us( + backend_capture_timestamp: Option, + fallback_wall_time_us: u64, + read_wall_time_us: u64, +) -> u64 { + backend_capture_timestamp + .and_then(|timestamp| u64::try_from(timestamp.as_micros()).ok()) + .and_then(|timestamp_us| validate_capture_timestamp_us(timestamp_us, read_wall_time_us)) + .unwrap_or(fallback_wall_time_us) +} + +fn i32_from_u32(value: u32, label: &'static str) -> Result { + i32::try_from(value).map_err(|_| DeviceVideoSourceError::InvalidFrame(label)) +} + +fn i32_from_usize(value: usize, label: &'static str) -> Result { + i32::try_from(value).map_err(|_| DeviceVideoSourceError::InvalidFrame(label)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum V4lTimestampClock { + Unknown, + Monotonic, + Copy, + Unsupported, +} + +fn v4l_timestamp_clock(flags: V4lBufferFlags) -> V4lTimestampClock { + let timestamp_type = flags.bits() & V4lBufferFlags::TIMESTAMP_MASK.bits(); + if timestamp_type == V4lBufferFlags::TIMESTAMP_MONOTONIC.bits() { + V4lTimestampClock::Monotonic + } else if timestamp_type == V4lBufferFlags::TIMESTAMP_COPY.bits() { + V4lTimestampClock::Copy + } else if timestamp_type == V4lBufferFlags::TIMESTAMP_UNKNOWN.bits() { + V4lTimestampClock::Unknown + } else { + V4lTimestampClock::Unsupported + } +} + +fn v4l_timestamp_to_wallclock( + timestamp: v4l::Timestamp, + clock: V4lTimestampClock, +) -> Option { + let frame_timestamp = Duration::from(timestamp); + if frame_timestamp.is_zero() { + return None; + } + + let monotonic_now = clock_time(libc::CLOCK_MONOTONIC)?; + let wall_now = clock_time(libc::CLOCK_REALTIME)?; + timestamp_to_wallclock(frame_timestamp, clock, monotonic_now, wall_now) +} + +fn timestamp_to_wallclock( + frame_timestamp: Duration, + clock: V4lTimestampClock, + monotonic_now: Duration, + wall_now: Duration, +) -> Option { + if frame_timestamp.is_zero() { + return None; + } + + match clock { + V4lTimestampClock::Monotonic => { + monotonic_timestamp_to_wallclock(frame_timestamp, monotonic_now, wall_now) + } + V4lTimestampClock::Unknown => { + monotonic_timestamp_to_wallclock(frame_timestamp, monotonic_now, wall_now) + .or(Some(frame_timestamp)) + } + V4lTimestampClock::Copy | V4lTimestampClock::Unsupported => None, + } +} + +fn monotonic_timestamp_to_wallclock( + frame_timestamp: Duration, + monotonic_now: Duration, + wall_now: Duration, +) -> Option { + let frame_age = monotonic_now.checked_sub(frame_timestamp)?; + wall_now.checked_sub(frame_age) +} + +fn clock_time(clock_id: libc::clockid_t) -> Option { + let mut time = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `time` is a valid out pointer and `clock_id` is supplied by libc constants. + let ret = unsafe { libc::clock_gettime(clock_id, &mut time) }; + if ret != 0 || time.tv_sec < 0 || time.tv_nsec < 0 { + return None; + } + + Some(Duration::new(time.tv_sec as u64, time.tv_nsec as u32)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sources::device::timestamp::MAX_CAPTURE_TIMESTAMP_AGE_US; + + #[test] + fn source_formats_exclude_unconvertible_ones() { + assert!(!is_supported_source_format(DeviceFrameFormat::I420)); + assert!(!is_supported_source_format(DeviceFrameFormat::Bgra)); + assert!(!is_supported_source_format(DeviceFrameFormat::Uyvy)); + assert!(is_supported_source_format(DeviceFrameFormat::Yuyv)); + } + + #[test] + fn frame_formats_for_request_prefers_the_requested_format() { + let request = DeviceFormatRequest::Exact(DeviceFormat::new( + VideoResolution::new(1280, 720), + 30, + DeviceFrameFormat::Mjpeg, + )); + let formats = frame_formats_for_request(&request); + assert_eq!(formats.first(), Some(&DeviceFrameFormat::Mjpeg)); + assert_eq!(formats.len(), default_frame_formats().len()); + } + + #[test] + fn frame_formats_for_request_pins_explicit_constraints() { + let request = DeviceFormatRequest::HighestFramerate { + resolution: None, + frame_format: Some(DeviceFrameFormat::Grey), + }; + assert_eq!(frame_formats_for_request(&request), vec![DeviceFrameFormat::Grey]); + } + + #[test] + fn ignores_stream_relative_capture_timestamp() { + // A small timestamp (relative to stream start rather than a clock) + // fails wall-clock validation and falls back to the read time. + let selected = + select_capture_wall_time_us(Some(Duration::from_micros(5)), 9_000_000, 10_000_000); + assert_eq!(selected, 9_000_000); + } + + #[test] + fn accepts_recent_backend_capture_timestamp() { + let selected = select_capture_wall_time_us( + Some(Duration::from_micros(9_999_000)), + 9_000_000, + 10_000_000, + ); + assert_eq!(selected, 9_999_000); + } + + #[test] + fn ignores_backend_capture_timestamp_older_than_max_age() { + let read_wall_time_us = 10_000_000 + MAX_CAPTURE_TIMESTAMP_AGE_US; + let selected = select_capture_wall_time_us( + Some(Duration::from_micros(10_000_000 - 1)), + 9_000_000, + read_wall_time_us, + ); + assert_eq!(selected, 9_000_000); + } + + #[test] + fn converts_monotonic_v4l_timestamp_to_wallclock() { + let converted = timestamp_to_wallclock( + Duration::from_secs(90), + V4lTimestampClock::Monotonic, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, Some(Duration::from_secs(990))); + } + + #[test] + fn infers_unknown_v4l_timestamp_clock() { + // Convertible as monotonic: treated as monotonic. + let converted = timestamp_to_wallclock( + Duration::from_secs(90), + V4lTimestampClock::Unknown, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, Some(Duration::from_secs(990))); + + // Ahead of the monotonic clock: passed through as-is. + let converted = timestamp_to_wallclock( + Duration::from_secs(500), + V4lTimestampClock::Unknown, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, Some(Duration::from_secs(500))); + } + + #[test] + fn rejects_copied_and_unsupported_v4l_timestamps() { + for clock in [V4lTimestampClock::Copy, V4lTimestampClock::Unsupported] { + let converted = timestamp_to_wallclock( + Duration::from_secs(90), + clock, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, None); + } + } + + #[test] + fn framerate_from_fraction_rounds_fractional_intervals() { + assert_eq!(framerate_from_fraction(1, 30), Some(30)); + assert_eq!(framerate_from_fraction(1001, 30000), Some(30)); + assert_eq!(framerate_from_fraction(1001, 60000), Some(60)); + } + + #[test] + fn framerate_from_fraction_rejects_zero_terms() { + assert_eq!(framerate_from_fraction(0, 30), None); + assert_eq!(framerate_from_fraction(30, 0), None); + } +} diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 98f28638d..584969e49 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -311,6 +311,12 @@ impl GStreamerVideoSource { source.pending_sample = Some(sample); } + log::info!( + "GStreamer pipeline ready: {:?} {} ({} resolution)", + source.sample_format.codec(), + source.resolution, + if config.resolution.is_none() { "discovered" } else { "declared" }, + ); Ok(source) } diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index 017f27e44..f743c92d6 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -15,5 +15,8 @@ #[cfg(feature = "demo")] pub mod demo; +#[cfg(feature = "device")] +pub mod device; + #[cfg(feature = "gstreamer")] pub mod gstreamer; diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index 1eb7962ba..abaa12005 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -20,10 +20,12 @@ __rustls-tls = ["livekit/__rustls-tls"] tracing = ["tokio/tracing", "console-subscriber"] # Capture sources (livekit-capture): publish tracks from server-side -# producers such as GStreamer pipelines. Links system GStreamer. +# producers such as camera devices or GStreamer pipelines. Links system +# GStreamer. capture = [ "dep:livekit-capture", "livekit-capture/demo", + "livekit-capture/device", "livekit-capture/gstreamer", "livekit-capture/tokio", ] diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 8ab77c595..be84f6481 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -80,6 +80,99 @@ message DemoVideoSourceConfig { required uint32 framerate_fps = 2; } +// Frame format delivered by a capture device. +enum DeviceFrameFormat { + DEVICE_FRAME_FORMAT_I420 = 0; + DEVICE_FRAME_FORMAT_NV12 = 1; + DEVICE_FRAME_FORMAT_BGRA = 2; + DEVICE_FRAME_FORMAT_RGB24 = 3; + DEVICE_FRAME_FORMAT_BGR24 = 4; + DEVICE_FRAME_FORMAT_YUYV = 5; + DEVICE_FRAME_FORMAT_UYVY = 6; + DEVICE_FRAME_FORMAT_GREY = 7; + DEVICE_FRAME_FORMAT_MJPEG = 8; +} + +// Capture format offered by or requested from a device. +message DeviceFormat { + // Frame dimensions. + required VideoSourceResolution resolution = 1; + // Frame rate in frames per second. + required uint32 framerate_fps = 2; + // Frame format. + required DeviceFrameFormat frame_format = 3; +} + +// Format selection requested from a capture device. The device negotiates +// the delivered format; CaptureSourceInfo reports the outcome. +message DeviceFormatRequest { + // Prefer the highest frame rate, optionally constrained. + message HighestFramerate { + optional VideoSourceResolution resolution = 1; + optional DeviceFrameFormat frame_format = 2; + } + // Prefer the highest resolution, optionally constrained. + message HighestResolution { + optional uint32 framerate_fps = 1; + optional DeviceFrameFormat frame_format = 2; + } + // The device's default format when unset. + oneof request { + // Require an exact format match. + DeviceFormat exact = 1; + // Use the device's closest supported format. + DeviceFormat closest = 2; + HighestFramerate highest_framerate = 3; + HighestResolution highest_resolution = 4; + } +} + +// Camera device capture using the platform's native capture stack. +message DeviceVideoSourceConfig { + // Device to capture from; the platform default device when unset. + oneof device { + // Position in the platform enumeration order. + uint32 device_index = 1; + // Platform-stable identifier, as reported by CaptureDeviceInfo.id. + string device_id = 2; + } + // Format requested from the device; the device default when unset. + optional DeviceFormatRequest format = 3; +} + +// Video capture device discovered by ListCaptureDevicesRequest. +message CaptureDeviceInfo { + // Platform-stable device identifier. + required string id = 1; + // Human-readable device name. + required string name = 2; + // Device model identifier, when available. + optional string model_id = 3; + // Device manufacturer, when available. + optional string manufacturer = 4; + // Capture formats reported by the device. + repeated DeviceFormat formats = 5; + // Whether `formats` is a complete list; some platforms do not enumerate + // formats up front. + required bool formats_complete = 6; +} + +message CaptureDeviceList { repeated CaptureDeviceInfo devices = 1; } + +// List the video capture devices available on this machine. +// +// Completes asynchronously with a ListCaptureDevicesCallback: enumeration +// queries the platform capture stack and may block briefly. +message ListCaptureDevicesRequest { optional uint64 request_async_id = 1; } +message ListCaptureDevicesResponse { required uint64 async_id = 1; } +message ListCaptureDevicesCallback { + required uint64 async_id = 1; + oneof message { + string error = 2; + CaptureDeviceList devices = 3; + } +} + // Kind of media a capture source produces. enum CaptureSourceKind { // Pixel frames, published through the WebRTC encoder. @@ -117,6 +210,7 @@ message NewCaptureSourceRequest { oneof config { GstreamerVideoSourceConfig gstreamer = 1; DemoVideoSourceConfig demo = 2; + DeviceVideoSourceConfig device = 4; } optional uint64 request_async_id = 3; } diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index b23cd0b98..971801216 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -189,8 +189,9 @@ message FfiRequest { NewCaptureSourceRequest new_capture_source = 87; StartCaptureRequest start_capture = 88; StopCaptureRequest stop_capture = 89; + ListCaptureDevicesRequest list_capture_devices = 90; - // NEXT_ID: 90 + // NEXT_ID: 91 } } @@ -323,8 +324,9 @@ message FfiResponse { NewCaptureSourceResponse new_capture_source = 87; StartCaptureResponse start_capture = 88; StopCaptureResponse stop_capture = 89; + ListCaptureDevicesResponse list_capture_devices = 90; - // NEXT_ID: 90 + // NEXT_ID: 91 } } @@ -396,8 +398,9 @@ message FfiEvent { // Capture sources (livekit-capture; requires the `capture` feature) NewCaptureSourceCallback new_capture_source = 47; CaptureSourceEvent capture_source_event = 48; + ListCaptureDevicesCallback list_capture_devices = 49; - // NEXT_ID: 49 + // NEXT_ID: 50 } } diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index bc7d844c0..9e8c35205 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -18,6 +18,10 @@ use livekit_capture::{ primitive::VideoResolution, sources::{ demo::DemoVideoSourceConfig, + device::{ + DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, DeviceSelector, + DeviceVideoSourceConfig, + }, gstreamer::{GStreamerBitrateUnit, GStreamerRateControlConfig, GStreamerVideoSourceConfig}, }, }; @@ -66,6 +70,110 @@ pub fn video_codec_to_proto(codec: EncodedVideoCodec) -> Option DeviceFrameFormat { + match format { + proto::DeviceFrameFormat::I420 => DeviceFrameFormat::I420, + proto::DeviceFrameFormat::Nv12 => DeviceFrameFormat::Nv12, + proto::DeviceFrameFormat::Bgra => DeviceFrameFormat::Bgra, + proto::DeviceFrameFormat::Rgb24 => DeviceFrameFormat::Rgb24, + proto::DeviceFrameFormat::Bgr24 => DeviceFrameFormat::Bgr24, + proto::DeviceFrameFormat::Yuyv => DeviceFrameFormat::Yuyv, + proto::DeviceFrameFormat::Uyvy => DeviceFrameFormat::Uyvy, + proto::DeviceFrameFormat::Grey => DeviceFrameFormat::Grey, + proto::DeviceFrameFormat::Mjpeg => DeviceFrameFormat::Mjpeg, + } +} + +fn device_frame_format_to_proto(format: DeviceFrameFormat) -> Option { + match format { + DeviceFrameFormat::I420 => Some(proto::DeviceFrameFormat::I420), + DeviceFrameFormat::Nv12 => Some(proto::DeviceFrameFormat::Nv12), + DeviceFrameFormat::Bgra => Some(proto::DeviceFrameFormat::Bgra), + DeviceFrameFormat::Rgb24 => Some(proto::DeviceFrameFormat::Rgb24), + DeviceFrameFormat::Bgr24 => Some(proto::DeviceFrameFormat::Bgr24), + DeviceFrameFormat::Yuyv => Some(proto::DeviceFrameFormat::Yuyv), + DeviceFrameFormat::Uyvy => Some(proto::DeviceFrameFormat::Uyvy), + DeviceFrameFormat::Grey => Some(proto::DeviceFrameFormat::Grey), + DeviceFrameFormat::Mjpeg => Some(proto::DeviceFrameFormat::Mjpeg), + // The frame format enum is non-exhaustive; formats unknown to the + // protocol are simply not reported. + _ => None, + } +} + +fn decode_device_frame_format(value: i32) -> FfiResult { + proto::DeviceFrameFormat::try_from(value) + .map(device_frame_format_from_proto) + .map_err(|_| FfiError::InvalidRequest("invalid device frame format".into())) +} + +fn device_format_from_proto(format: proto::DeviceFormat) -> FfiResult { + Ok(DeviceFormat { + resolution: format.resolution.into(), + framerate_fps: format.framerate_fps, + frame_format: decode_device_frame_format(format.frame_format)?, + }) +} + +fn device_format_to_proto(format: DeviceFormat) -> Option { + Some(proto::DeviceFormat { + resolution: proto::VideoSourceResolution { + width: format.resolution.width, + height: format.resolution.height, + }, + framerate_fps: format.framerate_fps, + frame_format: device_frame_format_to_proto(format.frame_format)?.into(), + }) +} + +fn device_format_request_from_proto( + request: proto::DeviceFormatRequest, +) -> FfiResult { + use proto::device_format_request::Request; + Ok(match request.request { + None => DeviceFormatRequest::Default, + Some(Request::Exact(format)) => { + DeviceFormatRequest::Exact(device_format_from_proto(format)?) + } + Some(Request::Closest(format)) => { + DeviceFormatRequest::Closest(device_format_from_proto(format)?) + } + Some(Request::HighestFramerate(constraint)) => DeviceFormatRequest::HighestFramerate { + resolution: constraint.resolution.map(VideoResolution::from), + frame_format: constraint.frame_format.map(decode_device_frame_format).transpose()?, + }, + Some(Request::HighestResolution(constraint)) => DeviceFormatRequest::HighestResolution { + framerate_fps: constraint.framerate_fps, + frame_format: constraint.frame_format.map(decode_device_frame_format).transpose()?, + }, + }) +} + +pub fn device_config_from_proto( + config: proto::DeviceVideoSourceConfig, +) -> FfiResult { + use proto::device_video_source_config::Device; + let device = match config.device { + None => DeviceSelector::Default, + Some(Device::DeviceIndex(index)) => DeviceSelector::Index(index as usize), + Some(Device::DeviceId(id)) => DeviceSelector::Id(id), + }; + let format = + config.format.map(device_format_request_from_proto).transpose()?.unwrap_or_default(); + Ok(DeviceVideoSourceConfig { device, format }) +} + +pub fn device_info_to_proto(info: DeviceInfo) -> proto::CaptureDeviceInfo { + proto::CaptureDeviceInfo { + id: info.id, + name: info.name, + model_id: info.model_id, + manufacturer: info.manufacturer, + formats: info.formats.into_iter().filter_map(device_format_to_proto).collect(), + formats_complete: info.formats_complete, + } +} + pub fn gstreamer_config_from_proto( config: proto::GstreamerVideoSourceConfig, ) -> FfiResult { diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index 58e6dbc5d..af00b667b 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -23,13 +23,20 @@ use livekit_capture::{ encoded::{EncodedVideoPump, EncodedVideoSource}, pixel::{PixelVideoPump, PixelVideoSource}, pump::{PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, - sources::{demo::DemoVideoSource, gstreamer::GStreamerVideoSource}, + sources::{ + demo::DemoVideoSource, + device::{self, DeviceVideoSource}, + gstreamer::GStreamerVideoSource, + }, }; use parking_lot::Mutex; use super::{video_source::FfiVideoSource, FfiHandle, FfiServer}; use crate::{ - conversion::capture::{gstreamer_config_from_proto, video_codec_to_proto}, + conversion::capture::{ + device_config_from_proto, device_info_to_proto, gstreamer_config_from_proto, + video_codec_to_proto, + }, proto, FfiError, FfiHandleId, FfiResult, }; @@ -118,6 +125,13 @@ async fn create_capture_source( let source: Box = Box::new(source); CapturePump::Pixel(PixelVideoPump::new(source)) } + proto::new_capture_source_request::Config::Device(config) => { + let source = DeviceVideoSource::new(device_config_from_proto(config)?) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Pixel(PixelVideoPump::new(source)) + } }; let (kind, resolution, codec, publish_options, rtc_source, stop) = match &pump { @@ -286,6 +300,27 @@ pub fn on_stop_capture( Ok(proto::StopCaptureResponse { error: None }) } +pub fn on_list_capture_devices( + server: &'static FfiServer, + request: proto::ListCaptureDevicesRequest, +) -> FfiResult { + let async_id = server.resolve_async_id(request.request_async_id); + server.async_runtime.spawn(async move { + let message = match device::devices().await { + Ok(devices) => { + proto::list_capture_devices_callback::Message::Devices(proto::CaptureDeviceList { + devices: devices.into_iter().map(device_info_to_proto).collect(), + }) + } + Err(err) => proto::list_capture_devices_callback::Message::Error(err.to_string()), + }; + let _ = server.send_event(proto::ffi_event::Message::ListCaptureDevices( + proto::ListCaptureDevicesCallback { async_id, message: Some(message) }, + )); + }); + Ok(proto::ListCaptureDevicesResponse { async_id }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 920d40872..5782bb4ff 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -1490,8 +1490,13 @@ pub fn handle_request( Request::StartCapture(req) => capture::on_start_capture(server, req)?.into(), #[cfg(feature = "capture")] Request::StopCapture(req) => capture::on_stop_capture(server, req)?.into(), + #[cfg(feature = "capture")] + Request::ListCaptureDevices(req) => capture::on_list_capture_devices(server, req)?.into(), #[cfg(not(feature = "capture"))] - Request::NewCaptureSource(_) | Request::StartCapture(_) | Request::StopCapture(_) => { + Request::NewCaptureSource(_) + | Request::StartCapture(_) + | Request::StopCapture(_) + | Request::ListCaptureDevices(_) => { return Err(FfiError::InvalidRequest( "livekit-ffi was built without the 'capture' feature".into(), )); From 3978d1087bf409ee3751855f355a4c3506f2cd44 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:03:44 +0000 Subject: [PATCH 37/56] generated protobuf --- livekit-ffi-node-bindings/proto/ffi_pb.d.ts | 20 +++++++++++++++++++- livekit-ffi-node-bindings/proto/ffi_pb.js | 5 ++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts index 12cb91bbe..0ffa8cb94 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts @@ -28,7 +28,7 @@ import type { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, Registe import type { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } from "./track_publication_pb.js"; import type { ByteStreamOpenCallback, ByteStreamOpenRequest, ByteStreamOpenResponse, ByteStreamReaderEvent, ByteStreamReaderReadAllCallback, ByteStreamReaderReadAllRequest, ByteStreamReaderReadAllResponse, ByteStreamReaderReadIncrementalRequest, ByteStreamReaderReadIncrementalResponse, ByteStreamReaderWriteToFileCallback, ByteStreamReaderWriteToFileRequest, ByteStreamReaderWriteToFileResponse, ByteStreamWriterCloseCallback, ByteStreamWriterCloseRequest, ByteStreamWriterCloseResponse, ByteStreamWriterWriteCallback, ByteStreamWriterWriteRequest, ByteStreamWriterWriteResponse, StreamSendBytesCallback, StreamSendBytesRequest, StreamSendBytesResponse, StreamSendFileCallback, StreamSendFileRequest, StreamSendFileResponse, StreamSendTextCallback, StreamSendTextRequest, StreamSendTextResponse, TextStreamOpenCallback, TextStreamOpenRequest, TextStreamOpenResponse, TextStreamReaderEvent, TextStreamReaderReadAllCallback, TextStreamReaderReadAllRequest, TextStreamReaderReadAllResponse, TextStreamReaderReadIncrementalRequest, TextStreamReaderReadIncrementalResponse, TextStreamWriterCloseCallback, TextStreamWriterCloseRequest, TextStreamWriterCloseResponse, TextStreamWriterWriteCallback, TextStreamWriterWriteRequest, TextStreamWriterWriteResponse } from "./data_stream_pb.js"; import type { DataTrackStreamEvent, DataTrackStreamReadRequest, DataTrackStreamReadResponse, DefineSchemaCallback, DefineSchemaRequest, DefineSchemaResponse, GetSchemaCallback, GetSchemaRequest, GetSchemaResponse, LocalDataTrackIsPublishedRequest, LocalDataTrackIsPublishedResponse, LocalDataTrackTryPushRequest, LocalDataTrackTryPushResponse, LocalDataTrackUnpublishRequest, LocalDataTrackUnpublishResponse, PublishDataTrackCallback, PublishDataTrackRequest, PublishDataTrackResponse, RemoteDataTrackIsPublishedRequest, RemoteDataTrackIsPublishedResponse, RemoteDataTrackSetPipelineOptionsRequest, RemoteDataTrackSetPipelineOptionsResponse, SubscribeDataTrackRequest, SubscribeDataTrackResponse } from "./data_track_pb.js"; -import type { CaptureSourceEvent, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } from "./capture_pb.js"; +import type { CaptureSourceEvent, ListCaptureDevicesCallback, ListCaptureDevicesRequest, ListCaptureDevicesResponse, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } from "./capture_pb.js"; /** * @generated from enum livekit.proto.LogLevel @@ -632,6 +632,12 @@ export declare class FfiRequest extends Message { */ value: StopCaptureRequest; case: "stopCapture"; + } | { + /** + * @generated from field: livekit.proto.ListCaptureDevicesRequest list_capture_devices = 90; + */ + value: ListCaptureDevicesRequest; + case: "listCaptureDevices"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); @@ -1214,6 +1220,12 @@ export declare class FfiResponse extends Message { */ value: StopCaptureResponse; case: "stopCapture"; + } | { + /** + * @generated from field: livekit.proto.ListCaptureDevicesResponse list_capture_devices = 90; + */ + value: ListCaptureDevicesResponse; + case: "listCaptureDevices"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); @@ -1536,6 +1548,12 @@ export declare class FfiEvent extends Message { */ value: CaptureSourceEvent; case: "captureSourceEvent"; + } | { + /** + * @generated from field: livekit.proto.ListCaptureDevicesCallback list_capture_devices = 49; + */ + value: ListCaptureDevicesCallback; + case: "listCaptureDevices"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.js b/livekit-ffi-node-bindings/proto/ffi_pb.js index c61a091f6..0755806d5 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.js +++ b/livekit-ffi-node-bindings/proto/ffi_pb.js @@ -30,7 +30,7 @@ const { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMe const { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } = require("./track_publication_pb.js"); const { ByteStreamOpenCallback, ByteStreamOpenRequest, ByteStreamOpenResponse, ByteStreamReaderEvent, ByteStreamReaderReadAllCallback, ByteStreamReaderReadAllRequest, ByteStreamReaderReadAllResponse, ByteStreamReaderReadIncrementalRequest, ByteStreamReaderReadIncrementalResponse, ByteStreamReaderWriteToFileCallback, ByteStreamReaderWriteToFileRequest, ByteStreamReaderWriteToFileResponse, ByteStreamWriterCloseCallback, ByteStreamWriterCloseRequest, ByteStreamWriterCloseResponse, ByteStreamWriterWriteCallback, ByteStreamWriterWriteRequest, ByteStreamWriterWriteResponse, StreamSendBytesCallback, StreamSendBytesRequest, StreamSendBytesResponse, StreamSendFileCallback, StreamSendFileRequest, StreamSendFileResponse, StreamSendTextCallback, StreamSendTextRequest, StreamSendTextResponse, TextStreamOpenCallback, TextStreamOpenRequest, TextStreamOpenResponse, TextStreamReaderEvent, TextStreamReaderReadAllCallback, TextStreamReaderReadAllRequest, TextStreamReaderReadAllResponse, TextStreamReaderReadIncrementalRequest, TextStreamReaderReadIncrementalResponse, TextStreamWriterCloseCallback, TextStreamWriterCloseRequest, TextStreamWriterCloseResponse, TextStreamWriterWriteCallback, TextStreamWriterWriteRequest, TextStreamWriterWriteResponse } = require("./data_stream_pb.js"); const { DataTrackStreamEvent, DataTrackStreamReadRequest, DataTrackStreamReadResponse, DefineSchemaCallback, DefineSchemaRequest, DefineSchemaResponse, GetSchemaCallback, GetSchemaRequest, GetSchemaResponse, LocalDataTrackIsPublishedRequest, LocalDataTrackIsPublishedResponse, LocalDataTrackTryPushRequest, LocalDataTrackTryPushResponse, LocalDataTrackUnpublishRequest, LocalDataTrackUnpublishResponse, PublishDataTrackCallback, PublishDataTrackRequest, PublishDataTrackResponse, RemoteDataTrackIsPublishedRequest, RemoteDataTrackIsPublishedResponse, RemoteDataTrackSetPipelineOptionsRequest, RemoteDataTrackSetPipelineOptionsResponse, SubscribeDataTrackRequest, SubscribeDataTrackResponse } = require("./data_track_pb.js"); -const { CaptureSourceEvent, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } = require("./capture_pb.js"); +const { CaptureSourceEvent, ListCaptureDevicesCallback, ListCaptureDevicesRequest, ListCaptureDevicesResponse, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } = require("./capture_pb.js"); /** * @generated from enum livekit.proto.LogLevel @@ -143,6 +143,7 @@ const FfiRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 87, name: "new_capture_source", kind: "message", T: NewCaptureSourceRequest, oneof: "message" }, { no: 88, name: "start_capture", kind: "message", T: StartCaptureRequest, oneof: "message" }, { no: 89, name: "stop_capture", kind: "message", T: StopCaptureRequest, oneof: "message" }, + { no: 90, name: "list_capture_devices", kind: "message", T: ListCaptureDevicesRequest, oneof: "message" }, ], ); @@ -241,6 +242,7 @@ const FfiResponse = /*@__PURE__*/ proto2.makeMessageType( { no: 87, name: "new_capture_source", kind: "message", T: NewCaptureSourceResponse, oneof: "message" }, { no: 88, name: "start_capture", kind: "message", T: StartCaptureResponse, oneof: "message" }, { no: 89, name: "stop_capture", kind: "message", T: StopCaptureResponse, oneof: "message" }, + { no: 90, name: "list_capture_devices", kind: "message", T: ListCaptureDevicesResponse, oneof: "message" }, ], ); @@ -301,6 +303,7 @@ const FfiEvent = /*@__PURE__*/ proto2.makeMessageType( { no: 46, name: "get_schema", kind: "message", T: GetSchemaCallback, oneof: "message" }, { no: 47, name: "new_capture_source", kind: "message", T: NewCaptureSourceCallback, oneof: "message" }, { no: 48, name: "capture_source_event", kind: "message", T: CaptureSourceEvent, oneof: "message" }, + { no: 49, name: "list_capture_devices", kind: "message", T: ListCaptureDevicesCallback, oneof: "message" }, ], ); From 254c7d38f0a86fa92f7a0672d80aaa597c71dbbb Mon Sep 17 00:00:00 2001 From: stavied <40528896+stephen-derosa@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:21:29 -0600 Subject: [PATCH 38/56] =?UTF-8?q?potential=20fix:=20av=20foundation=20take?= =?UTF-8?q?=20the=20duration=20from=20the=20matched=20range=E2=80=A6=20(#1?= =?UTF-8?q?314)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/sources/device/avfoundation.rs | 78 +++++++++++++++---- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/livekit-capture/src/sources/device/avfoundation.rs b/livekit-capture/src/sources/device/avfoundation.rs index e6d5ec457..76702b968 100644 --- a/livekit-capture/src/sources/device/avfoundation.rs +++ b/livekit-capture/src/sources/device/avfoundation.rs @@ -32,7 +32,7 @@ use livekit::webrtc::video_frame::{ }; use objc2::rc::Retained; use objc2::runtime::{AnyObject, ProtocolObject}; -use objc2::{define_class, msg_send, AnyThread, DefinedClass, Message}; +use objc2::{define_class, msg_send, sel, AnyThread, DefinedClass, Message}; use objc2_av_foundation::{ AVCaptureDevice, AVCaptureDeviceFormat, AVCaptureDeviceInput, AVCaptureOutput, AVCaptureSession, AVCaptureSessionPreset1280x720, AVCaptureSessionPreset1920x1080, @@ -413,21 +413,34 @@ fn configure_input_frame_duration( let Some(framerate) = requested_framerate(request).filter(|framerate| *framerate > 0) else { return; }; - // SAFETY: `input` is the live input just added to the session. The - // support predicate is checked before setting the locked duration. + + // AVCaptureDeviceInput's locked-frame-duration API is macOS 26.0+, while + // the SDK builds against an older deployment target. Sending a selector the + // running OS does not implement raises an Objective-C exception, which Rust + // cannot catch and which therefore aborts the process, so probe first. + if !input.respondsToSelector(sel!(isLockedVideoFrameDurationSupported)) + || !input.respondsToSelector(sel!(setActiveLockedVideoFrameDuration:)) + { + return; + } + + // SAFETY: `input` is the live input just added to the session, and the + // selector was confirmed present above. if !unsafe { input.isLockedVideoFrameDurationSupported() } { return; } - let duration = unsafe { CMTime::with_seconds(1.0 / framerate as f64, 600) }; - // SAFETY: `device` and `input` belong to the same session setup path. - // The requested rate has already been checked against the active format - // before the device frame durations are set, and `input` reports locked - // frame duration support. + // SAFETY: `device` and `input` belong to the same session setup path, and + // reading activeFormat is valid once the input has been added. + let duration = unsafe { device_format_frame_duration(&device.activeFormat(), framerate) }; + let Some(duration) = duration else { + return; + }; + + // SAFETY: `input` reports locked frame duration support, and `duration` + // came from a frame-rate range of the device's active format. unsafe { - if device_format_supports_framerate(&device.activeFormat(), framerate) { - input.setActiveLockedVideoFrameDuration(duration); - } + input.setActiveLockedVideoFrameDuration(duration); } } @@ -903,15 +916,46 @@ fn device_format_resolution(format: &AVCaptureDeviceFormat) -> Option bool { + device_format_frame_duration(format, framerate).is_some() +} + +/// Frame duration to apply for `framerate` on `format`, or `None` when no +/// frame-rate range covers it. +/// +/// The duration is taken from the matched range's own bounds instead of being +/// derived from `framerate` alone. AVFoundation raises an Objective-C +/// exception — which aborts the process, since Rust cannot catch it — for any +/// duration outside a range's `[minFrameDuration, maxFrameDuration]`, and +/// devices commonly advertise near-integral rates whose exact duration is not +/// the reciprocal of the rounded rate. A UVC camera reporting 30.00003 fps +/// accepts 1/30.00003 s but rejects 1/30 s. +fn device_format_frame_duration(format: &AVCaptureDeviceFormat, framerate: u32) -> Option { let requested = framerate as f64; // SAFETY: `format` is an AVCaptureDeviceFormat from the device's immutable formats array. // The returned frame-rate ranges are immutable AVFoundation objects. - unsafe { format.videoSupportedFrameRateRanges() }.iter().any(|range| { + unsafe { format.videoSupportedFrameRateRanges() }.iter().find_map(|range| { // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. let min = unsafe { range.minFrameRate() }; // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. let max = unsafe { range.maxFrameRate() }; - requested >= min.floor() && requested <= max.ceil() + if requested < min.floor() || requested > max.ceil() { + return None; + } + // Rate and duration are inverses, so the slowest rate carries the + // longest duration. Snapping to an endpoint keeps a rounded request + // inside the bounds the format actually accepts. + Some(if requested <= min { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + unsafe { range.maxFrameDuration() } + } else if requested >= max { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + unsafe { range.minFrameDuration() } + } else { + // The rate lies strictly inside the range, so its reciprocal lies + // strictly inside the range's duration bounds. + // SAFETY: `requested` is finite and greater than zero here. + unsafe { CMTime::with_seconds(1.0 / requested, 600) } + }) }) } @@ -980,12 +1024,12 @@ fn configure_locked_device( // SAFETY: The caller holds the configuration lock, and reading activeFormat is valid. None => unsafe { device.activeFormat() }, }; - if !device_format_supports_framerate(&active_format, framerate) { + let Some(duration) = device_format_frame_duration(&active_format, framerate) else { return Ok(()); - } + }; - let duration = unsafe { CMTime::with_seconds(1.0 / framerate as f64, 600) }; - // SAFETY: The device is locked for configuration and the CMTime value is finite. + // SAFETY: The device is locked for configuration and `duration` came from a + // frame-rate range of the format now active on the device. unsafe { device.setActiveVideoMinFrameDuration(duration); device.setActiveVideoMaxFrameDuration(duration); From b0636d6486def81b69ed4d035821fb22e9920dde Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:05:25 -0700 Subject: [PATCH 39/56] Avoid dynamic allocation --- livekit-capture/src/sources/demo.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index ab8cdb8b5..5d87a23e9 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -61,7 +61,7 @@ pub struct DemoVideoSourceConfig { pub struct DemoVideoSource { config: DemoVideoSourceConfig, /// One `(y, u, v)` sample triple per palette color. - colors: Vec<(u8, u8, u8)>, + colors: [(u8, u8, u8); PALETTE.len()], started: Option, frame_index: u64, } @@ -77,7 +77,7 @@ impl DemoVideoSource { return Err(SourceError::new(DemoVideoSourceConfigError::ZeroFramerate)); } - let colors = PALETTE.iter().map(|&color| yuv_from_rgb(color)).collect(); + let colors = PALETTE.map(yuv_from_rgb); Ok(Self { config, colors, started: None, frame_index: 0 }) } From 468e332efb6ef1955cd7e2bc8edc358eeaa4cf27 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:05:51 -0700 Subject: [PATCH 40/56] Avoid hand rolled ceiling division --- livekit-capture/src/sources/gstreamer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 584969e49..17366b049 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -135,7 +135,7 @@ impl GStreamerBitrateUnit { fn property_value(self, target_bitrate_bps: u64) -> u64 { match self { Self::BitsPerSecond => target_bitrate_bps, - Self::KilobitsPerSecond => target_bitrate_bps.saturating_add(999) / 1000, + Self::KilobitsPerSecond => target_bitrate_bps.div_ceil(1000), } } } From 8c270c59b6e72d3d80319be4364464842b658ccb Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:09:15 -0700 Subject: [PATCH 41/56] Introduce helper for run blocking --- livekit-capture/src/lib.rs | 1 + livekit-capture/src/sources/device/mod.rs | 12 ++------- livekit-capture/src/sources/gstreamer.rs | 6 +---- livekit-capture/src/utils.rs | 30 +++++++++++++++++++++++ 4 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 livekit-capture/src/utils.rs diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 7d0c62f12..408c87fdc 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -18,3 +18,4 @@ pub mod pixel; pub mod primitive; pub mod pump; pub mod sources; +mod utils; diff --git a/livekit-capture/src/sources/device/mod.rs b/livekit-capture/src/sources/device/mod.rs index 645a53e54..616974d40 100644 --- a/livekit-capture/src/sources/device/mod.rs +++ b/livekit-capture/src/sources/device/mod.rs @@ -261,11 +261,7 @@ impl DeviceInfo { /// non-async form. #[cfg(feature = "tokio")] pub async fn devices() -> Result, SourceError> { - match tokio::task::spawn_blocking(devices_blocking).await { - Ok(result) => result, - Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), - Err(err) => Err(SourceError::new(err)), - } + crate::utils::run_blocking(devices_blocking).await } /// Lists the video capture devices available on this machine. @@ -318,11 +314,7 @@ impl DeviceVideoSource { /// [`DeviceVideoSource::new_blocking`] for everything else. #[cfg(feature = "tokio")] pub async fn new(config: DeviceVideoSourceConfig) -> Result { - match tokio::task::spawn_blocking(move || Self::new_blocking(config)).await { - Ok(result) => result, - Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), - Err(err) => Err(SourceError::new(err)), - } + crate::utils::run_blocking(move || Self::new_blocking(config)).await } /// Opens the configured device and negotiates the capture format. diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 17366b049..84393447c 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -220,11 +220,7 @@ impl GStreamerVideoSource { /// [`GStreamerVideoSource::new_blocking`] for everything else. #[cfg(feature = "tokio")] pub async fn new(config: GStreamerVideoSourceConfig) -> Result { - match tokio::task::spawn_blocking(move || Self::new_blocking(config)).await { - Ok(result) => result, - Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), - Err(err) => Err(SourceError::new(err)), - } + crate::utils::run_blocking(move || Self::new_blocking(config)).await } /// Builds, owns, and starts a GStreamer pipeline from configuration. diff --git a/livekit-capture/src/utils.rs b/livekit-capture/src/utils.rs new file mode 100644 index 000000000..19cab29d5 --- /dev/null +++ b/livekit-capture/src/utils.rs @@ -0,0 +1,30 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Crate-internal utilities. + +/// Runs blocking source construction or enumeration on the tokio blocking +/// pool, resuming panics on the caller and surfacing join failures as source +/// errors. +#[cfg(feature = "tokio")] +#[allow(dead_code)] +pub(crate) async fn run_blocking( + task: impl FnOnce() -> Result + Send + 'static, +) -> Result { + match tokio::task::spawn_blocking(task).await { + Ok(result) => result, + Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), + Err(err) => Err(crate::error::SourceError::new(err)), + } +} From cb7c2ba1154c642a7370c4873b8c0497b6f97316 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:11:13 -0700 Subject: [PATCH 42/56] Don't return useless result --- livekit-capture/src/encoded/h26x.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index da26c026c..5a9a31bec 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -354,13 +354,12 @@ pub fn annex_b_nal_ranges(bytes: &[u8]) -> Vec> { } /// Returns borrowed NAL units from an Annex-B buffer. -pub fn annex_b_nalus(bytes: &[u8]) -> Result, CaptureError> { - let nals = annex_b_nal_ranges(bytes) +pub fn annex_b_nalus(bytes: &[u8]) -> Vec<&[u8]> { + annex_b_nal_ranges(bytes) .into_iter() .map(|range| &bytes[range]) .filter(|nal| !nal.is_empty()) - .collect::>(); - Ok(nals) + .collect() } /// Creates an Annex-B access unit from H.264/AVC length-prefixed NAL units. @@ -409,7 +408,7 @@ pub fn access_unit_from_nalus( /// Returns true when an Annex-B access unit contains an intra/key picture. pub fn is_keyframe_annex_b(codec: EncodedVideoCodec, bytes: &[u8]) -> Result { - let nals = annex_b_nalus(bytes)?; + let nals = annex_b_nalus(bytes); is_keyframe_nalus(codec, &nals) } @@ -599,7 +598,7 @@ mod tests { #[test] fn splits_annex_b_nals_with_three_and_four_byte_prefixes() { let bytes = [0, 0, 1, 0x67, 1, 0, 0, 0, 1, 0x65, 2, 3]; - let nals = annex_b_nalus(&bytes).unwrap(); + let nals = annex_b_nalus(&bytes); assert_eq!(nals, vec![&[0x67, 1][..], &[0x65, 2, 3][..]]); } From 3411e2e5b5dad6b53942b08772fcada434a483e6 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:12:00 -0700 Subject: [PATCH 43/56] Expose error description --- livekit-capture/src/pump.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index 6ce950238..a4fa235e2 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -40,10 +40,10 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum PumpError { /// The capture source failed. - #[error("capture source failed")] + #[error("capture source failed: {0}")] Source(#[from] SourceError), /// The RTC source rejected a frame. - #[error("frame capture failed")] + #[error("frame capture failed: {0}")] Capture(#[from] CaptureError), /// The pump thread panicked. #[error("pump panicked: {0}")] From 39d5f9fc26694f9ec272bf6f93ccb4049a8ec066 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:14:38 -0700 Subject: [PATCH 44/56] Avoid redundant work --- livekit-capture/src/sources/device/v4l2.rs | 24 +++++++--------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/livekit-capture/src/sources/device/v4l2.rs b/livekit-capture/src/sources/device/v4l2.rs index f6518c9c9..a099210c9 100644 --- a/livekit-capture/src/sources/device/v4l2.rs +++ b/livekit-capture/src/sources/device/v4l2.rs @@ -586,17 +586,17 @@ fn device_capture_format(device: &Device) -> Result<(DeviceFormat, u32), DeviceV fn enumerate_device_formats(device: &Device) -> Result, DeviceVideoSourceError> { let mut formats = Vec::new(); - let fourccs = device - .enum_formats() - .map_err(backend_error)? - .into_iter() - .filter_map(|format| frame_format_from_fourcc(format.fourcc).map(|_| format.fourcc)) - .collect::>(); + let mut seen_fourccs = Vec::new(); - for fourcc in dedup_fourccs(fourccs) { + for description in device.enum_formats().map_err(backend_error)? { + let fourcc = description.fourcc; let Some(frame_format) = frame_format_from_fourcc(fourcc) else { continue; }; + if seen_fourccs.contains(&fourcc) { + continue; + } + seen_fourccs.push(fourcc); let frame_sizes = device.enum_framesizes(fourcc).map_err(backend_error)?; for resolution in frame_sizes.into_iter().flat_map(resolutions_from_frame_size) { let intervals = device @@ -635,16 +635,6 @@ fn frame_format_from_fourcc(fourcc: FourCC) -> Option { } } -fn dedup_fourccs(fourccs: Vec) -> Vec { - let mut deduped = Vec::new(); - for fourcc in fourccs { - if !deduped.contains(&fourcc) { - deduped.push(fourcc); - } - } - deduped -} - fn resolutions_from_frame_size(size: v4l::FrameSize) -> Vec { match size.size { FrameSizeEnum::Discrete(discrete) => { From bce0f8d5bdc43dc1aad88f974a8a50434baee44a Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:42:59 -0700 Subject: [PATCH 45/56] Reduce API surface area - Remove code not needed for presently implemented sources - Make internal were possible - Remove dead error variants --- livekit-capture/src/encoded/h26x.rs | 9 +- livekit-capture/src/encoded/mod.rs | 344 ++------------------- livekit-capture/src/encoded/pump.rs | 31 +- livekit-capture/src/error.rs | 11 +- livekit-capture/src/pixel/pump.rs | 15 +- livekit-capture/src/sources/demo.rs | 4 +- livekit-capture/src/sources/device/v4l2.rs | 6 +- livekit-capture/src/sources/gstreamer.rs | 72 ++--- 8 files changed, 65 insertions(+), 427 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index 5a9a31bec..4dac57bfa 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -14,8 +14,8 @@ use crate::{ encoded::{ - annex_b_payload, h264_nal_type, h265_nal_type, is_keyframe_nalus, CodecSpecific, - EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, + annex_b_payload, h264_nal_type, h265_nal_type, is_keyframe_nalus, EncodedFrameType, + EncodedVideoCodec, OwnedEncodedAccessUnit, }, error::CaptureError, primitive::VideoResolution, @@ -389,10 +389,7 @@ pub fn access_unit_from_annex_b( } else { EncodedFrameType::Delta }; - let mut access_unit = - OwnedEncodedAccessUnit::new(codec, payload, timestamp_us, frame_type, resolution); - access_unit.codec_specific = CodecSpecific::default_for(codec); - Ok(access_unit) + Ok(OwnedEncodedAccessUnit::new(codec, payload, timestamp_us, frame_type, resolution)) } /// Creates an Annex-B access unit from raw NAL units. diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index 88c05f347..b38ffcf46 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -77,34 +77,7 @@ pub trait EncodedVideoSource: Send { fn update_rate_control(&mut self, _target: EncodedRateControl) {} } -/// Encoded byte-stream framing used by encoded source backends. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum EncodedWireFormat { - /// H.264 Annex-B byte stream. - H264AnnexB, - /// H.264/AVC byte stream with length-prefixed NAL units. - /// - /// `nal_length_size` is the number of big-endian length bytes before each NAL unit. Values - /// from 1 through 4 are accepted; 4 is the common AVC configuration. - H264Avc { - /// Length-prefix size in bytes. - nal_length_size: u8, - }, - /// H.265 Annex-B byte stream. - H265AnnexB, - /// RTP packets for the supplied codec and RTP clock rate. - Rtp { - /// RTP payload codec. - codec: EncodedVideoCodec, - /// RTP timestamp clock rate. - clock_rate: u32, - }, - /// MPEG transport stream carrying encoded video. - MpegTs, -} - -/// Encoded video codec carried by an [`EncodedAccessUnit`]. +/// Encoded video codec carried by an [`OwnedEncodedAccessUnit`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg_attr( feature = "serde", @@ -135,150 +108,6 @@ pub enum EncodedFrameType { Delta, } -/// Layer identifiers associated with an encoded frame. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct EncodedLayerInfo { - /// Spatial layer index, when present. - pub spatial_id: Option, - /// Temporal layer index, when present. - pub temporal_id: Option, -} - -/// H.264 packetization mode for passthrough metadata. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum H264PacketizationMode { - /// Non-interleaved packetization mode. - NonInterleaved, -} - -/// Codec-specific metadata for encoded passthrough. -#[derive(Debug, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum CodecSpecific { - /// No codec-specific metadata. - None, - /// H.264-specific metadata. - H264 { - /// H.264 RTP packetization mode. - packetization_mode: H264PacketizationMode, - }, - /// H.265-specific metadata. - H265, - /// VP8-specific metadata. - VP8 { - /// Temporal layer index, when present. - temporal_id: Option, - /// Whether this frame synchronizes a temporal layer. - layer_sync: bool, - }, - /// VP9-specific metadata. - VP9 { - /// Temporal layer index, when present. - temporal_id: Option, - /// Spatial layer index, when present. - spatial_id: Option, - /// Whether this frame depends on an inter-layer reference. - inter_layer_predicted: Option, - }, - /// AV1-specific metadata. - AV1 { - /// RTP scalability mode, such as `L1T1`. - scalability_mode: Option, - /// Encoded dependency descriptor bytes, when supplied by the caller. - dependency_descriptor: Option>, - }, -} - -impl Default for CodecSpecific { - fn default() -> Self { - Self::None - } -} - -impl CodecSpecific { - /// Returns the single-layer default metadata for a codec, matching what - /// the passthrough encoder synthesizes on the wire. - pub fn default_for(codec: EncodedVideoCodec) -> Self { - match codec { - EncodedVideoCodec::H264 => { - Self::H264 { packetization_mode: H264PacketizationMode::NonInterleaved } - } - EncodedVideoCodec::H265 => Self::H265, - EncodedVideoCodec::VP8 => Self::VP8 { temporal_id: None, layer_sync: false }, - EncodedVideoCodec::VP9 => { - Self::VP9 { temporal_id: None, spatial_id: None, inter_layer_predicted: None } - } - EncodedVideoCodec::AV1 => { - Self::AV1 { scalability_mode: Some("L1T1".to_owned()), dependency_descriptor: None } - } - } - } -} - -/// Borrowed encoded payload fragment. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct EncodedFragment<'a> { - /// Encoded fragment bytes. - pub bytes: &'a [u8], -} - -/// Encoded access-unit payload. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum EncodedPayload<'a> { - /// One contiguous payload buffer. - Contiguous(&'a [u8]), - /// Multiple payload fragments. - Fragments(&'a [EncodedFragment<'a>]), - /// Owned payload bytes. - Owned(Vec), -} - -impl EncodedPayload<'_> { - pub(crate) fn is_empty(&self) -> bool { - match self { - Self::Contiguous(bytes) => bytes.is_empty(), - Self::Fragments(fragments) => { - fragments.is_empty() || fragments.iter().any(|fragment| fragment.bytes.is_empty()) - } - Self::Owned(bytes) => bytes.is_empty(), - } - } - - pub(crate) fn to_vec(&self) -> Vec { - match self { - Self::Contiguous(bytes) => bytes.to_vec(), - Self::Fragments(fragments) => { - let len = fragments.iter().map(|fragment| fragment.bytes.len()).sum(); - let mut payload = Vec::with_capacity(len); - for fragment in *fragments { - payload.extend_from_slice(fragment.bytes); - } - payload - } - Self::Owned(bytes) => bytes.clone(), - } - } -} - -/// One encoded video access unit. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EncodedAccessUnit<'a> { - /// Encoded codec. - pub codec: EncodedVideoCodec, - /// Encoded payload. - pub payload: EncodedPayload<'a>, - /// Capture timestamp in microseconds. - pub timestamp_us: i64, - /// Encoded frame type. - pub frame_type: EncodedFrameType, - /// Encoded frame resolution in pixels. - pub resolution: VideoResolution, - /// Optional layer identifiers. - pub layers: EncodedLayerInfo, - /// Optional codec-specific metadata. - pub codec_specific: CodecSpecific, -} - /// Owned encoded video access unit. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OwnedEncodedAccessUnit { @@ -292,10 +121,6 @@ pub struct OwnedEncodedAccessUnit { pub frame_type: EncodedFrameType, /// Encoded frame resolution in pixels. pub resolution: VideoResolution, - /// Optional layer identifiers. - pub layers: EncodedLayerInfo, - /// Optional codec-specific metadata. - pub codec_specific: CodecSpecific, } impl OwnedEncodedAccessUnit { @@ -307,98 +132,7 @@ impl OwnedEncodedAccessUnit { frame_type: EncodedFrameType, resolution: VideoResolution, ) -> Self { - Self { - codec, - payload: payload.into(), - timestamp_us, - frame_type, - resolution, - layers: EncodedLayerInfo::default(), - codec_specific: CodecSpecific::None, - } - } - - /// Borrows this owned access unit as an [`EncodedAccessUnit`]. - pub fn as_access_unit(&self) -> EncodedAccessUnit<'_> { - EncodedAccessUnit { - codec: self.codec, - payload: EncodedPayload::Contiguous(&self.payload), - timestamp_us: self.timestamp_us, - frame_type: self.frame_type, - resolution: self.resolution, - layers: self.layers, - codec_specific: self.codec_specific.clone(), - } - } - - /// Creates an owned access unit by copying a borrowed access unit. - pub fn copy_from(access_unit: &EncodedAccessUnit<'_>) -> Self { - Self { - codec: access_unit.codec, - payload: Bytes::from(access_unit.payload.to_vec()), - timestamp_us: access_unit.timestamp_us, - frame_type: access_unit.frame_type, - resolution: access_unit.resolution, - layers: access_unit.layers, - codec_specific: access_unit.codec_specific.clone(), - } - } -} - -impl<'a> EncodedAccessUnit<'a> { - /// Creates an access unit from one contiguous payload. - pub fn contiguous( - codec: EncodedVideoCodec, - payload: &'a [u8], - timestamp_us: i64, - frame_type: EncodedFrameType, - resolution: VideoResolution, - ) -> Self { - Self { - codec, - payload: EncodedPayload::Contiguous(payload), - timestamp_us, - frame_type, - resolution, - layers: EncodedLayerInfo::default(), - codec_specific: CodecSpecific::None, - } - } - - /// Creates an H.264 access unit from raw NAL-unit payloads. - pub fn from_h264_nalus( - nal_units: &[&[u8]], - timestamp_us: i64, - resolution: VideoResolution, - ) -> Result, CaptureError> { - Self::from_nalus(EncodedVideoCodec::H264, nal_units, timestamp_us, resolution) - } - - /// Creates an H.265 access unit from raw NAL-unit payloads. - pub fn from_h265_nalus( - nal_units: &[&[u8]], - timestamp_us: i64, - resolution: VideoResolution, - ) -> Result, CaptureError> { - Self::from_nalus(EncodedVideoCodec::H265, nal_units, timestamp_us, resolution) - } - - fn from_nalus( - codec: EncodedVideoCodec, - nal_units: &[&[u8]], - timestamp_us: i64, - resolution: VideoResolution, - ) -> Result, CaptureError> { - let is_key = is_keyframe_nalus(codec, nal_units)?; - Ok(EncodedAccessUnit { - codec, - payload: EncodedPayload::Owned(annex_b_payload(nal_units)?), - timestamp_us, - frame_type: if is_key { EncodedFrameType::Key } else { EncodedFrameType::Delta }, - resolution, - layers: EncodedLayerInfo::default(), - codec_specific: CodecSpecific::default_for(codec), - }) + Self { codec, payload: payload.into(), timestamp_us, frame_type, resolution } } } @@ -539,81 +273,45 @@ mod tests { use super::*; #[test] - fn h264_nal_helper_assembles_annex_b_and_detects_keyframe() { + fn h264_keyframe_requires_idr_nal() { let sps = [0x67, 1, 2, 3]; let idr = [0x65, 4, 5, 6]; - let au = - EncodedAccessUnit::from_h264_nalus(&[&sps, &idr], 10, VideoResolution::new(640, 480)) - .unwrap(); - - assert_eq!(au.codec, EncodedVideoCodec::H264); - assert_eq!(au.frame_type, EncodedFrameType::Key); - assert_eq!( - au.payload, - EncodedPayload::Owned(vec![0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5, 6]) - ); + let non_idr = [0x61, 1, 2]; + + assert!(is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &idr]).unwrap()); + assert!(!is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &non_idr]).unwrap()); } #[test] - fn h265_nal_helper_requires_parameter_sets_and_idr_keyframe() { + fn h265_keyframe_requires_parameter_sets_and_idr() { let vps = [0x40, 1, 2]; let sps = [0x42, 1, 2]; let pps = [0x44, 1, 2]; let idr_w_radl = [19 << 1, 1, 3]; - let idr_without_headers = EncodedAccessUnit::from_h265_nalus( - &[&vps, &idr_w_radl], - 10, - VideoResolution::new(640, 480), - ) - .unwrap(); - let key = EncodedAccessUnit::from_h265_nalus( - &[&vps, &sps, &pps, &idr_w_radl], - 10, - VideoResolution::new(640, 480), - ) - .unwrap(); let cra = [21 << 1, 1, 3]; - let cra_with_headers = EncodedAccessUnit::from_h265_nalus( - &[&vps, &sps, &pps, &cra], - 10, - VideoResolution::new(640, 480), - ) - .unwrap(); - - assert_eq!(idr_without_headers.codec, EncodedVideoCodec::H265); - assert_eq!(idr_without_headers.frame_type, EncodedFrameType::Delta); - assert_eq!(key.frame_type, EncodedFrameType::Key); - assert_eq!(cra_with_headers.frame_type, EncodedFrameType::Delta); + + assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &idr_w_radl]).unwrap()); + assert!( + is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &idr_w_radl]).unwrap() + ); + assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &cra]).unwrap()); } #[test] fn h265_rejects_too_short_nal_header() { - let err = - EncodedAccessUnit::from_h265_nalus(&[&[0x26]], 10, VideoResolution::new(640, 480)) - .unwrap_err(); + let err = is_keyframe_nalus(EncodedVideoCodec::H265, &[&[0x26]]).unwrap_err(); assert_eq!(err, CaptureError::H265NalTooShort); } #[test] - fn fragments_reject_empty_fragment() { - let fragments = [EncodedFragment { bytes: &[1] }, EncodedFragment { bytes: &[] }]; - let payload = EncodedPayload::Fragments(&fragments); - assert!(payload.is_empty()); + fn annex_b_payload_prefixes_each_nal_unit() { + let payload = annex_b_payload(&[&[0x67, 1, 2, 3], &[0x65, 4, 5, 6]]).unwrap(); + assert_eq!(payload, vec![0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5, 6]); } #[test] - fn owned_access_unit_borrows_without_copying_payload() { - let owned = OwnedEncodedAccessUnit::new( - EncodedVideoCodec::H264, - Bytes::from_static(&[1, 2, 3]), - 10, - EncodedFrameType::Delta, - VideoResolution::new(640, 480), - ); - - let borrowed = owned.as_access_unit(); - assert_eq!(borrowed.codec, EncodedVideoCodec::H264); - assert_eq!(borrowed.payload, EncodedPayload::Contiguous(&[1, 2, 3])); - assert_eq!(borrowed.timestamp_us, 10); + fn annex_b_payload_rejects_empty_input() { + assert_eq!(annex_b_payload(&[]).unwrap_err(), CaptureError::EmptyPayload); + assert_eq!(annex_b_payload(&[&[]]).unwrap_err(), CaptureError::EmptyPayload); } } diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs index b6af2f204..95e4d77be 100644 --- a/livekit-capture/src/encoded/pump.rs +++ b/livekit-capture/src/encoded/pump.rs @@ -16,10 +16,7 @@ //! source. use crate::{ - encoded::{ - CodecSpecific, EncodedFrameType, EncodedLayerInfo, EncodedVideoSource, - OwnedEncodedAccessUnit, - }, + encoded::{EncodedFrameType, EncodedVideoSource, OwnedEncodedAccessUnit}, error::CaptureError, pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, }; @@ -165,7 +162,9 @@ fn capture_access_unit( access_unit: &OwnedEncodedAccessUnit, frame_metadata: Option, ) -> Result<(), CaptureError> { - validate_access_unit(access_unit)?; + if access_unit.payload.is_empty() { + return Err(CaptureError::EmptyPayload); + } let frame = EncodedVideoFrame { codec: access_unit.codec.into(), @@ -178,28 +177,6 @@ fn capture_access_unit( rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) } -/// The passthrough path forwards single-layer streams: access units carrying -/// temporal/spatial layer ids or layering metadata are rejected so callers -/// are not misled into thinking that metadata reaches the wire. -fn validate_access_unit(access_unit: &OwnedEncodedAccessUnit) -> Result<(), CaptureError> { - if access_unit.payload.is_empty() { - return Err(CaptureError::EmptyPayload); - } - if access_unit.layers != EncodedLayerInfo::default() { - return Err(CaptureError::UnsupportedLayeredEncoding( - "temporal/spatial layer ids are not forwarded by the passthrough encoder", - )); - } - if access_unit.codec_specific != CodecSpecific::None - && access_unit.codec_specific != CodecSpecific::default_for(access_unit.codec) - { - return Err(CaptureError::UnsupportedLayeredEncoding( - "codec-specific layering metadata is not forwarded by the passthrough encoder", - )); - } - Ok(()) -} - #[cfg(test)] mod tests { use std::collections::VecDeque; diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index c72255817..8c7a4621a 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -14,7 +14,7 @@ //! Error types shared across capture paths. -use crate::encoded::{EncodedVideoCodec, EncodedWireFormat}; +use crate::encoded::EncodedVideoCodec; use std::{error::Error as StdError, fmt}; use thiserror::Error; @@ -54,21 +54,12 @@ pub enum CaptureError { /// H.265 NAL unit is too short to contain its header. #[error("H.265 NAL unit is too short")] H265NalTooShort, - /// Access unit carries layering metadata the passthrough cannot forward. - #[error("unsupported layered encoding: {0}")] - UnsupportedLayeredEncoding(&'static str), /// Codec is represented by the API but not yet supported by native passthrough. #[error("encoded passthrough does not support {0:?} yet")] UnsupportedCodec(EncodedVideoCodec), /// Encoded payload or transport data is malformed. #[error("invalid encoded data: {0}")] InvalidEncodedData(&'static str), - /// Wire format is represented by the API but not supported by this source. - #[error("encoded wire format is not supported by this source: {0:?}")] - UnsupportedWireFormat(EncodedWireFormat), - /// Capture backend is not available on this platform. - #[error("{0} is not supported on this platform")] - UnsupportedPlatform(&'static str), /// The underlying source rejected the frame. #[error("capture source rejected the frame")] CaptureFailed, diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs index 7bb98329f..58487db31 100644 --- a/livekit-capture/src/pixel/pump.rs +++ b/livekit-capture/src/pixel/pump.rs @@ -240,7 +240,10 @@ mod tests { RESOLUTION } - fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { + fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { panic!("source exploded"); } } @@ -264,7 +267,10 @@ mod tests { RESOLUTION } - fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { + fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { std::thread::sleep(std::time::Duration::from_millis(1)); Ok(Some(pixel_frame(0))) } @@ -289,7 +295,10 @@ mod tests { RESOLUTION } - fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { + fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { std::thread::sleep(std::time::Duration::from_millis(1)); Ok(Some(pixel_frame(0))) } diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index 5d87a23e9..7251ed2c9 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -14,7 +14,9 @@ //! Solid-color demo video source for testing. -use crate::{error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop}; +use crate::{ + error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop, +}; use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; use std::{ thread, diff --git a/livekit-capture/src/sources/device/v4l2.rs b/livekit-capture/src/sources/device/v4l2.rs index a099210c9..a636d8976 100644 --- a/livekit-capture/src/sources/device/v4l2.rs +++ b/livekit-capture/src/sources/device/v4l2.rs @@ -128,11 +128,7 @@ impl Session { // negotiated format actually delivers frames. let first_frame = session.read_frame()?; session.pending_frame = Some(first_frame); - log::info!( - "Opened device \"{}\": {} (converted to I420)", - device_name, - session.format, - ); + log::info!("Opened device \"{}\": {} (converted to I420)", device_name, session.format,); Ok(session) } diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 84393447c..f51c60e0d 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -22,8 +22,7 @@ use thiserror::Error; use crate::{ encoded::{ h26x::{access_unit_from_annex_b, access_unit_from_h264_avc}, - CodecSpecific, EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, - OwnedEncodedAccessUnit, + EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, OwnedEncodedAccessUnit, }, error::{CaptureError, SourceError}, primitive::VideoResolution, @@ -33,8 +32,7 @@ use livekit::webrtc::video_source::EncodedRateControl; /// Encoded sample format expected from a GStreamer appsink. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum GStreamerSampleFormat { +enum GStreamerSampleFormat { /// H.264 Annex-B access units, usually from `h264parse` with byte-stream caps. H264AnnexB, /// H.264 access units with AVC length-prefixed NAL units. @@ -53,7 +51,7 @@ pub enum GStreamerSampleFormat { impl GStreamerSampleFormat { /// Returns the encoded codec carried by this sample format. - pub fn codec(self) -> EncodedVideoCodec { + fn codec(self) -> EncodedVideoCodec { match self { Self::H264AnnexB => EncodedVideoCodec::H264, Self::H264Avc { .. } => EncodedVideoCodec::H264, @@ -295,8 +293,10 @@ impl GStreamerVideoSource { // verified lazily against the first sample instead. if config.resolution.is_none() { let sample = source.wait_first_sample().map_err(SourceError::new)?; - let caps = - sample.caps().ok_or(GStreamerVideoSourceError::MissingResolutionCaps).map_err(SourceError::new)?; + let caps = sample + .caps() + .ok_or(GStreamerVideoSourceError::MissingResolutionCaps) + .map_err(SourceError::new)?; source.resolution = resolution_from_caps(caps) .ok_or(GStreamerVideoSourceError::MissingResolutionCaps) .map_err(SourceError::new)?; @@ -319,7 +319,8 @@ impl GStreamerVideoSource { /// Blocks until the pipeline produces its first sample, surfacing bus /// errors and bounding the wait by the discovery timeout. fn wait_first_sample(&self) -> Result { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(DISCOVERY_TIMEOUT.seconds()); + let deadline = + std::time::Instant::now() + std::time::Duration::from_secs(DISCOVERY_TIMEOUT.seconds()); loop { self.check_bus()?; if let Some(sample) = self.appsink.try_pull_sample(SAMPLE_WAIT) { @@ -440,8 +441,7 @@ impl GStreamerVideoSource { fn timestamp_us(&mut self, buffer: &gst::BufferRef) -> i64 { if let Some(timestamp) = buffer.pts().or_else(|| buffer.dts()) { let timestamp_us = clock_time_to_timestamp_us(0, timestamp); - self.next_fallback_timestamp_us = - timestamp_us.saturating_add(self.frame_interval_us); + self.next_fallback_timestamp_us = timestamp_us.saturating_add(self.frame_interval_us); return timestamp_us; } @@ -577,9 +577,7 @@ pub enum GStreamerVideoSourceError { #[error("pipeline reached end of stream before producing a sample")] EndedBeforeFirstSample, /// Negotiated caps carry no resolution to discover. - #[error( - "negotiated caps declare no resolution; declare `resolution` in the configuration" - )] + #[error("negotiated caps declare no resolution; declare `resolution` in the configuration")] MissingResolutionCaps, /// The pipeline produces a different resolution than configured. #[error("pipeline produces {actual}, but the configuration declares {configured}")] @@ -648,15 +646,13 @@ fn access_unit_from_sample_payload( return Err(CaptureError::EmptyPayload); } - let mut access_unit = OwnedEncodedAccessUnit::new( + Ok(OwnedEncodedAccessUnit::new( codec, Bytes::copy_from_slice(payload), timestamp_us, frame_type, resolution, - ); - access_unit.codec_specific = CodecSpecific::default_for(codec); - Ok(access_unit) + )) } } } @@ -666,8 +662,7 @@ fn resolution_from_caps(caps: &gst::CapsRef) -> Option { let structure = caps.structure(0)?; let width = structure.get::("width").ok()?; let height = structure.get::("height").ok()?; - (width > 0 && height > 0) - .then(|| VideoResolution::new(width as u32, height as u32)) + (width > 0 && height > 0).then(|| VideoResolution::new(width as u32, height as u32)) } /// Derives the fallback frame interval from the caps framerate, when @@ -744,7 +739,7 @@ pub fn encoded_caps(codec: EncodedVideoCodec) -> Result GStreamerSampleFormat { +fn sample_format_for_codec(codec: EncodedVideoCodec) -> GStreamerSampleFormat { match codec { EncodedVideoCodec::H264 => GStreamerSampleFormat::H264AnnexB, EncodedVideoCodec::H265 => GStreamerSampleFormat::H265AnnexB, @@ -770,7 +765,7 @@ pub fn parser_name(codec: EncodedVideoCodec) -> Option<&'static str> { /// as-is (its sink caps decide the sample format). Otherwise the pipeline /// must leave one encoded video source pad unlinked; the codec parser, a /// capsfilter, and an appsink are created and linked to it. -pub fn ensure_encoded_appsink( +fn ensure_encoded_appsink( pipeline: &gst::Pipeline, requested_codec: Option, ) -> Result<(gst_app::AppSink, GStreamerSampleFormat), GStreamerPipelineError> { @@ -925,7 +920,7 @@ fn sample_format_from_pad_caps( } /// Infers the appsink sample format from a caps structure. -pub fn sample_format_from_caps_structure( +fn sample_format_from_caps_structure( structure: &gst::StructureRef, ) -> Result, GStreamerPipelineError> { let Some(codec) = codec_from_caps_name(structure.name()) else { @@ -979,19 +974,19 @@ fn h264_avc_nal_length_size_from_caps(structure: &gst::StructureRef) -> u8 { } /// Reads the AVC NAL length-prefix size from `avcC` codec data. -pub fn h264_avc_nal_length_size_from_codec_data(codec_data: &[u8]) -> Option { +fn h264_avc_nal_length_size_from_codec_data(codec_data: &[u8]) -> Option { let length_size = (codec_data.get(4)? & 0x03) + 1; (1..=4).contains(&length_size).then_some(length_size) } /// Infers the encoded codec advertised by a pad's caps. -pub fn codec_from_pad_caps(pad: &gst::Pad) -> Option { +fn codec_from_pad_caps(pad: &gst::Pad) -> Option { let caps = pad.current_caps().unwrap_or_else(|| pad.query_caps(None)); caps.iter().find_map(|structure| codec_from_caps_name(structure.name())) } /// Maps a caps media-type name to an encoded codec. -pub fn codec_from_caps_name(name: &str) -> Option { +fn codec_from_caps_name(name: &str) -> Option { match name { "video/x-h264" => Some(EncodedVideoCodec::H264), "video/x-h265" => Some(EncodedVideoCodec::H265), @@ -1051,33 +1046,6 @@ mod tests { assert_eq!(access_unit.codec, EncodedVideoCodec::VP8); assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); - assert_eq!( - access_unit.codec_specific, - CodecSpecific::VP8 { temporal_id: None, layer_sync: false } - ); - } - - #[test] - fn sample_payload_access_unit_sets_vp9_and_av1_specifics() { - let vp9 = access_unit_from_sample_payload( - GStreamerSampleFormat::AccessUnit { codec: EncodedVideoCodec::VP9 }, - &[1, 2, 3], - 2_000, - EncodedFrameType::Key, - VideoResolution::new(640, 480), - ) - .unwrap(); - assert_eq!(vp9.codec_specific, CodecSpecific::default_for(EncodedVideoCodec::VP9)); - - let av1 = access_unit_from_sample_payload( - GStreamerSampleFormat::AccessUnit { codec: EncodedVideoCodec::AV1 }, - &[1, 2, 3], - 2_000, - EncodedFrameType::Key, - VideoResolution::new(640, 480), - ) - .unwrap(); - assert_eq!(av1.codec_specific, CodecSpecific::default_for(EncodedVideoCodec::AV1)); } #[test] From cc69e2fb483cc29ed9c56c59a5b32cda252c0241 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:43:08 -0700 Subject: [PATCH 46/56] Add AGENTS.md --- livekit-capture/AGENTS.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 livekit-capture/AGENTS.md diff --git a/livekit-capture/AGENTS.md b/livekit-capture/AGENTS.md new file mode 100644 index 000000000..d2d247b85 --- /dev/null +++ b/livekit-capture/AGENTS.md @@ -0,0 +1,20 @@ +# AGENTS.md + +## Adding a source + +- Add a feature gate in `Cargo.toml` + - If the new source requires dependencies, they should only be include when feature is enabled +- Define a module for the new source in `src/sources/mod.rs`: + ```rust + #[cfg(feature = "my-source")] + pub mod my_source; + ``` +- Follow conventions for existing sources +- Source implementation should be self-contained in its module and submodules + - Only hoist functionality to a higher level module when shared by multiple sources +- Source (e.g, `MyVideoSource`) is constructed from a config struct (e.g., `MyVideoSourceConfig`) +- Add conditional derives for `serde` and `schemars` for config +- Implement `PixelVideoSource` or `EncodedVideoSource` for your source + - NEVER add a source that is not consumable uniformly through one of these traits + - Doing so would break API contract and break integration for consumers +- Keep API surface minimal and hide implementation details From b7d4b725e32438e8970b4961c4196fa446482af3 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:01:13 -0700 Subject: [PATCH 47/56] Keep error cases per source --- livekit-capture/src/encoded/h26x.rs | 245 +++++++++++++++++------ livekit-capture/src/encoded/mod.rs | 126 +----------- livekit-capture/src/encoded/pump.rs | 17 +- livekit-capture/src/error.rs | 29 +-- livekit-capture/src/pump.rs | 8 +- livekit-capture/src/sources/gstreamer.rs | 14 +- 6 files changed, 220 insertions(+), 219 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index 4dac57bfa..bc3d65489 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -12,16 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! H.264/H.265 parsing helpers: NAL-unit splitting, access-unit assembly and +//! delimiting, and keyframe detection for the encoded ingest paths. + use crate::{ - encoded::{ - annex_b_payload, h264_nal_type, h265_nal_type, is_keyframe_nalus, EncodedFrameType, - EncodedVideoCodec, OwnedEncodedAccessUnit, - }, - error::CaptureError, + encoded::{EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit}, primitive::VideoResolution, }; use bytes::Bytes; use std::ops::Range; +use thiserror::Error; + +/// Error returned by the H26x parsing helpers. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum H26xParseError { + /// Encoded payload is empty. + #[error("encoded payload is empty")] + EmptyPayload, + /// H.265 NAL unit is too short to contain its header. + #[error("H.265 NAL unit is too short")] + H265NalTooShort, + /// Codec is not supported by the H26x parsing helpers. + #[error("H26x parsing does not support {0:?}")] + UnsupportedCodec(EncodedVideoCodec), + /// Encoded payload or transport data is malformed. + #[error("invalid encoded data: {0}")] + InvalidEncodedData(&'static str), +} + +/// Start code prepended to each NAL unit when assembling Annex-B payloads. +const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; /// Upper bound on bytes buffered while waiting for an access-unit boundary. const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; @@ -34,15 +54,15 @@ const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; #[cfg(test)] pub(crate) trait AccessUnitParser { /// Appends bytes and returns the next complete access unit, if any. - fn push(&mut self, bytes: &[u8]) -> Result, CaptureError>; + fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError>; /// Returns the next complete access unit from already-buffered bytes. - fn drain(&mut self) -> Result, CaptureError> { + fn drain(&mut self) -> Result, H26xParseError> { self.push(&[]) } /// Flushes remaining buffered bytes as the final access unit. - fn flush(&mut self) -> Result, CaptureError>; + fn flush(&mut self) -> Result, H26xParseError>; } /// H26x Annex-B parser state. @@ -82,11 +102,11 @@ impl AnnexBAccessUnitParser { start_timestamp_us: i64, frame_interval_us: i64, resolution: VideoResolution, - ) -> Result { + ) -> Result { match codec { EncodedVideoCodec::H264 | EncodedVideoCodec::H265 => {} EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { - return Err(CaptureError::UnsupportedCodec(codec)); + return Err(H26xParseError::UnsupportedCodec(codec)); } } @@ -102,17 +122,17 @@ impl AnnexBAccessUnitParser { } /// Pushes encoded bytes and returns the next complete access unit if one is found. - pub fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { + pub fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError> { self.pending.extend_from_slice(bytes); self.drain_next(false) } /// Flushes the pending bytes as the final access unit. - pub fn flush(&mut self) -> Result, CaptureError> { + pub fn flush(&mut self) -> Result, H26xParseError> { self.drain_next(true) } - fn drain_next(&mut self, at_eof: bool) -> Result, CaptureError> { + fn drain_next(&mut self, at_eof: bool) -> Result, H26xParseError> { self.scan_pending(); if let Some(split_at) = @@ -124,7 +144,7 @@ impl AnnexBAccessUnitParser { return self.take_access_unit(self.pending.len()); } if !at_eof && self.pending.len() > MAX_PENDING_ACCESS_UNIT_BYTES { - return Err(CaptureError::InvalidEncodedData( + return Err(H26xParseError::InvalidEncodedData( "access unit exceeds maximum buffered size", )); } @@ -161,7 +181,7 @@ impl AnnexBAccessUnitParser { fn take_access_unit( &mut self, byte_len: usize, - ) -> Result, CaptureError> { + ) -> Result, H26xParseError> { if byte_len == 0 { return Ok(None); } @@ -191,11 +211,11 @@ impl AnnexBAccessUnitParser { #[cfg(test)] impl AccessUnitParser for AnnexBAccessUnitParser { - fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { + fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError> { AnnexBAccessUnitParser::push(self, bytes) } - fn flush(&mut self) -> Result, CaptureError> { + fn flush(&mut self) -> Result, H26xParseError> { AnnexBAccessUnitParser::flush(self) } } @@ -208,7 +228,7 @@ impl AvcAccessUnitParser { start_timestamp_us: i64, frame_interval_us: i64, resolution: VideoResolution, - ) -> Result { + ) -> Result { validate_avc_nal_length_size(nal_length_size)?; Ok(Self { @@ -226,17 +246,17 @@ impl AvcAccessUnitParser { pub(crate) fn push( &mut self, bytes: &[u8], - ) -> Result, CaptureError> { + ) -> Result, H26xParseError> { self.pending.extend_from_slice(bytes); self.drain_next(false) } /// Flushes the pending bytes as the final access unit. - pub(crate) fn flush(&mut self) -> Result, CaptureError> { + pub(crate) fn flush(&mut self) -> Result, H26xParseError> { self.drain_next(true) } - fn drain_next(&mut self, at_eof: bool) -> Result, CaptureError> { + fn drain_next(&mut self, at_eof: bool) -> Result, H26xParseError> { self.scan_pending(at_eof)?; if let Some(split_at) = avc_access_unit_split_index( @@ -250,7 +270,7 @@ impl AvcAccessUnitParser { return self.take_access_unit(self.pending.len()); } if !at_eof && self.pending.len() > MAX_PENDING_ACCESS_UNIT_BYTES { - return Err(CaptureError::InvalidEncodedData( + return Err(H26xParseError::InvalidEncodedData( "access unit exceeds maximum buffered size", )); } @@ -258,12 +278,12 @@ impl AvcAccessUnitParser { } /// Parses length-prefixed NAL units appended since the previous call. - fn scan_pending(&mut self, at_eof: bool) -> Result<(), CaptureError> { + fn scan_pending(&mut self, at_eof: bool) -> Result<(), H26xParseError> { let nal_length_size = self.nal_length_size as usize; while self.scan_cursor < self.pending.len() { if self.pending.len() - self.scan_cursor < nal_length_size { if at_eof { - return Err(CaptureError::InvalidEncodedData("truncated AVC NAL length")); + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL length")); } break; } @@ -271,15 +291,15 @@ impl AvcAccessUnitParser { let nal_start = self.scan_cursor + nal_length_size; let nal_len = read_avc_nal_length(&self.pending[self.scan_cursor..nal_start]); if nal_len == 0 { - return Err(CaptureError::InvalidEncodedData("empty AVC NAL unit")); + return Err(H26xParseError::InvalidEncodedData("empty AVC NAL unit")); } let Some(nal_end) = nal_start.checked_add(nal_len) else { - return Err(CaptureError::InvalidEncodedData("AVC NAL unit length overflow")); + return Err(H26xParseError::InvalidEncodedData("AVC NAL unit length overflow")); }; if nal_end > self.pending.len() { if at_eof { - return Err(CaptureError::InvalidEncodedData("truncated AVC NAL unit")); + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL unit")); } break; } @@ -293,7 +313,7 @@ impl AvcAccessUnitParser { fn take_access_unit( &mut self, byte_len: usize, - ) -> Result, CaptureError> { + ) -> Result, H26xParseError> { if byte_len == 0 { return Ok(None); } @@ -318,11 +338,11 @@ impl AvcAccessUnitParser { #[cfg(test)] impl AccessUnitParser for AvcAccessUnitParser { - fn push(&mut self, bytes: &[u8]) -> Result, CaptureError> { + fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError> { AvcAccessUnitParser::push(self, bytes) } - fn flush(&mut self) -> Result, CaptureError> { + fn flush(&mut self) -> Result, H26xParseError> { AvcAccessUnitParser::flush(self) } } @@ -368,7 +388,7 @@ pub fn access_unit_from_h264_avc( nal_length_size: u8, timestamp_us: i64, resolution: VideoResolution, -) -> Result { +) -> Result { let nals = avc_nalus(payload, nal_length_size)?; access_unit_from_nalus(EncodedVideoCodec::H264, &nals, timestamp_us, resolution) } @@ -379,9 +399,9 @@ pub fn access_unit_from_annex_b( payload: Bytes, timestamp_us: i64, resolution: VideoResolution, -) -> Result { +) -> Result { if payload.is_empty() { - return Err(CaptureError::EmptyPayload); + return Err(H26xParseError::EmptyPayload); } let frame_type = if is_keyframe_annex_b(codec, &payload)? { @@ -398,22 +418,90 @@ pub fn access_unit_from_nalus( nal_units: &[&[u8]], timestamp_us: i64, resolution: VideoResolution, -) -> Result { +) -> Result { let payload = Bytes::from(annex_b_payload(nal_units)?); access_unit_from_annex_b(codec, payload, timestamp_us, resolution) } /// Returns true when an Annex-B access unit contains an intra/key picture. -pub fn is_keyframe_annex_b(codec: EncodedVideoCodec, bytes: &[u8]) -> Result { +pub fn is_keyframe_annex_b(codec: EncodedVideoCodec, bytes: &[u8]) -> Result { let nals = annex_b_nalus(bytes); is_keyframe_nalus(codec, &nals) } +/// Returns true when the NAL units form a WebRTC-usable key frame. +fn is_keyframe_nalus( + codec: EncodedVideoCodec, + nal_units: &[&[u8]], +) -> Result { + match codec { + EncodedVideoCodec::H264 => { + nal_units.iter().try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == 5)) + } + EncodedVideoCodec::H265 => { + let mut has_vps = false; + let mut has_sps = false; + let mut has_pps = false; + let mut has_idr = false; + + for nal in nal_units { + match h265_nal_type(nal)? { + 32 => has_vps = true, + 33 => has_sps = true, + 34 => has_pps = true, + 19 | 20 => has_idr = true, + _ => {} + } + } + + Ok(has_vps && has_sps && has_pps && has_idr) + } + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + Err(H26xParseError::UnsupportedCodec(codec)) + } + } +} + +fn h264_nal_type(nal: &[u8]) -> Result { + let header = nal.first().ok_or(H26xParseError::EmptyPayload)?; + Ok(header & 0x1f) +} + +fn h265_nal_type(nal: &[u8]) -> Result { + if nal.is_empty() { + return Err(H26xParseError::EmptyPayload); + } + if nal.len() < 2 { + return Err(H26xParseError::H265NalTooShort); + } + Ok((nal[0] >> 1) & 0x3f) +} + +fn annex_b_payload(nal_units: &[&[u8]]) -> Result, H26xParseError> { + if nal_units.is_empty() { + return Err(H26xParseError::EmptyPayload); + } + let len = nal_units.iter().try_fold(0usize, |len, nal| { + if nal.is_empty() { + Err(H26xParseError::EmptyPayload) + } else { + Ok(len + ANNEX_B_START_CODE.len() + nal.len()) + } + })?; + + let mut payload = Vec::with_capacity(len); + for nal in nal_units { + payload.extend_from_slice(&ANNEX_B_START_CODE); + payload.extend_from_slice(nal); + } + Ok(payload) +} + fn access_unit_split_index( codec: EncodedVideoCodec, bytes: &[u8], ranges: &[Range], -) -> Result, CaptureError> { +) -> Result, H26xParseError> { match access_unit_boundary_nal(codec, bytes, ranges)? { Some(index) => split_start_code_index(bytes, ranges[index].start).map(Some), None => Ok(None), @@ -425,12 +513,12 @@ fn avc_access_unit_split_index( bytes: &[u8], ranges: &[Range], nal_length_size: usize, -) -> Result, CaptureError> { +) -> Result, H26xParseError> { match access_unit_boundary_nal(EncodedVideoCodec::H264, bytes, ranges)? { Some(index) => ranges[index] .start .checked_sub(nal_length_size) - .ok_or(CaptureError::InvalidEncodedData("missing AVC NAL length")) + .ok_or(H26xParseError::InvalidEncodedData("missing AVC NAL length")) .map(Some), None => Ok(None), } @@ -442,7 +530,7 @@ fn access_unit_boundary_nal( codec: EncodedVideoCodec, bytes: &[u8], ranges: &[Range], -) -> Result, CaptureError> { +) -> Result, H26xParseError> { let mut seen_vcl = false; for (index, range) in ranges.iter().enumerate() { let nal = &bytes[range.clone()]; @@ -465,7 +553,7 @@ fn min_nal_header_len(codec: EncodedVideoCodec) -> usize { } } -fn starts_new_access_unit(codec: EncodedVideoCodec, nal: &[u8]) -> Result { +fn starts_new_access_unit(codec: EncodedVideoCodec, nal: &[u8]) -> Result { Ok(match codec { EncodedVideoCodec::H264 => match h264_nal_type(nal)? { // Prefix SEI(6), SPS(7), PPS(8), and AUD(9) open a new access unit. @@ -489,27 +577,27 @@ fn starts_new_access_unit(codec: EncodedVideoCodec, nal: &[u8]) -> Result false, }, EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { - return Err(CaptureError::UnsupportedCodec(codec)); + return Err(H26xParseError::UnsupportedCodec(codec)); } }) } -fn split_start_code_index(bytes: &[u8], nal_start: usize) -> Result { +fn split_start_code_index(bytes: &[u8], nal_start: usize) -> Result { if nal_start >= 4 && bytes[nal_start - 4..nal_start] == [0, 0, 0, 1] { return Ok(nal_start - 4); } if nal_start >= 3 && bytes[nal_start - 3..nal_start] == [0, 0, 1] { return Ok(nal_start - 3); } - Err(CaptureError::InvalidEncodedData("missing Annex-B start code")) + Err(H26xParseError::InvalidEncodedData("missing Annex-B start code")) } -fn is_vcl_nal(codec: EncodedVideoCodec, nal: &[u8]) -> Result { +fn is_vcl_nal(codec: EncodedVideoCodec, nal: &[u8]) -> Result { Ok(match codec { EncodedVideoCodec::H264 => (1..=5).contains(&h264_nal_type(nal)?), EncodedVideoCodec::H265 => h265_nal_type(nal)? <= 31, EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { - return Err(CaptureError::UnsupportedCodec(codec)); + return Err(H26xParseError::UnsupportedCodec(codec)); } }) } @@ -528,10 +616,10 @@ fn find_start_code(bytes: &[u8]) -> Option<(usize, usize)> { None } -fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, CaptureError> { +fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, H26xParseError> { let ranges = avc_nal_ranges(payload, nal_length_size, true)?; if ranges.is_empty() { - return Err(CaptureError::EmptyPayload); + return Err(H26xParseError::EmptyPayload); } Ok(ranges.into_iter().map(|range| &payload[range]).collect()) } @@ -540,7 +628,7 @@ fn avc_nal_ranges( bytes: &[u8], nal_length_size: u8, at_eof: bool, -) -> Result>, CaptureError> { +) -> Result>, H26xParseError> { validate_avc_nal_length_size(nal_length_size)?; let nal_length_size = nal_length_size as usize; @@ -549,7 +637,7 @@ fn avc_nal_ranges( while cursor < bytes.len() { if bytes.len() - cursor < nal_length_size { if at_eof { - return Err(CaptureError::InvalidEncodedData("truncated AVC NAL length")); + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL length")); } break; } @@ -557,15 +645,15 @@ fn avc_nal_ranges( let nal_len = read_avc_nal_length(&bytes[cursor..cursor + nal_length_size]); cursor += nal_length_size; if nal_len == 0 { - return Err(CaptureError::InvalidEncodedData("empty AVC NAL unit")); + return Err(H26xParseError::InvalidEncodedData("empty AVC NAL unit")); } let Some(nal_end) = cursor.checked_add(nal_len) else { - return Err(CaptureError::InvalidEncodedData("AVC NAL unit length overflow")); + return Err(H26xParseError::InvalidEncodedData("AVC NAL unit length overflow")); }; if nal_end > bytes.len() { if at_eof { - return Err(CaptureError::InvalidEncodedData("truncated AVC NAL unit")); + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL unit")); } break; } @@ -581,17 +669,60 @@ fn read_avc_nal_length(bytes: &[u8]) -> usize { bytes.iter().fold(0usize, |len, byte| (len << 8) | usize::from(*byte)) } -fn validate_avc_nal_length_size(nal_length_size: u8) -> Result<(), CaptureError> { +fn validate_avc_nal_length_size(nal_length_size: u8) -> Result<(), H26xParseError> { if (1..=4).contains(&nal_length_size) { return Ok(()); } - Err(CaptureError::InvalidEncodedData("invalid AVC NAL length size")) + Err(H26xParseError::InvalidEncodedData("invalid AVC NAL length size")) } #[cfg(test)] mod tests { use super::*; + #[test] + fn h264_keyframe_requires_idr_nal() { + let sps = [0x67, 1, 2, 3]; + let idr = [0x65, 4, 5, 6]; + let non_idr = [0x61, 1, 2]; + + assert!(is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &idr]).unwrap()); + assert!(!is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &non_idr]).unwrap()); + } + + #[test] + fn h265_keyframe_requires_parameter_sets_and_idr() { + let vps = [0x40, 1, 2]; + let sps = [0x42, 1, 2]; + let pps = [0x44, 1, 2]; + let idr_w_radl = [19 << 1, 1, 3]; + let cra = [21 << 1, 1, 3]; + + assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &idr_w_radl]).unwrap()); + assert!( + is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &idr_w_radl]).unwrap() + ); + assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &cra]).unwrap()); + } + + #[test] + fn h265_rejects_too_short_nal_header() { + let err = is_keyframe_nalus(EncodedVideoCodec::H265, &[&[0x26]]).unwrap_err(); + assert_eq!(err, H26xParseError::H265NalTooShort); + } + + #[test] + fn annex_b_payload_prefixes_each_nal_unit() { + let payload = annex_b_payload(&[&[0x67, 1, 2, 3], &[0x65, 4, 5, 6]]).unwrap(); + assert_eq!(payload, vec![0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5, 6]); + } + + #[test] + fn annex_b_payload_rejects_empty_input() { + assert_eq!(annex_b_payload(&[]).unwrap_err(), H26xParseError::EmptyPayload); + assert_eq!(annex_b_payload(&[&[]]).unwrap_err(), H26xParseError::EmptyPayload); + } + #[test] fn splits_annex_b_nals_with_three_and_four_byte_prefixes() { let bytes = [0, 0, 1, 0x67, 1, 0, 0, 0, 1, 0x65, 2, 3]; @@ -630,7 +761,7 @@ mod tests { access_unit_from_h264_avc(&[0, 0, 0, 3, 0x65], 4, 10, VideoResolution::new(640, 480)) .unwrap_err(); - assert_eq!(err, CaptureError::InvalidEncodedData("truncated AVC NAL unit")); + assert_eq!(err, H26xParseError::InvalidEncodedData("truncated AVC NAL unit")); } #[test] @@ -939,7 +1070,7 @@ mod tests { let err = parser.push(&vec![0xff; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); assert_eq!( err, - CaptureError::InvalidEncodedData("access unit exceeds maximum buffered size") + H26xParseError::InvalidEncodedData("access unit exceeds maximum buffered size") ); } @@ -953,7 +1084,7 @@ mod tests { let err = parser.push(&vec![0x41; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); assert_eq!( err, - CaptureError::InvalidEncodedData("access unit exceeds maximum buffered size") + H26xParseError::InvalidEncodedData("access unit exceeds maximum buffered size") ); } } diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index b38ffcf46..5f31e3c23 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -21,11 +21,7 @@ //! implemented for `Box`, so sources can be constructed dynamically //! and driven through the same generic pump. -use crate::{ - error::{CaptureError, SourceError}, - primitive::VideoResolution, - pump::PumpStop, -}; +use crate::{error::SourceError, primitive::VideoResolution, pump::PumpStop}; use bytes::Bytes; use livekit::{ options::VideoCodec, @@ -41,8 +37,6 @@ pub mod h26x; mod pump; pub use pump::EncodedVideoPump; -const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; - /// Source of pre-encoded video access units, such as an encoding pipeline. pub trait EncodedVideoSource: Send { /// Nominal output resolution, used to size the RTC source. @@ -58,6 +52,9 @@ pub trait EncodedVideoSource: Send { /// integrate it into the blocking wait, or bound each wait so the token /// is observed within a frame interval or so. The pump distinguishes a /// stop from end of stream via the token. + /// + /// Access units must carry a non-empty payload; the pump reports a + /// violation as a source error. fn next_access_unit( &mut self, stop: &PumpStop, @@ -136,39 +133,6 @@ impl OwnedEncodedAccessUnit { } } -/// Returns true when the NAL units form a WebRTC-usable key frame. -pub(crate) fn is_keyframe_nalus( - codec: EncodedVideoCodec, - nal_units: &[&[u8]], -) -> Result { - match codec { - EncodedVideoCodec::H264 => { - nal_units.iter().try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == 5)) - } - EncodedVideoCodec::H265 => { - let mut has_vps = false; - let mut has_sps = false; - let mut has_pps = false; - let mut has_idr = false; - - for nal in nal_units { - match h265_nal_type(nal)? { - 32 => has_vps = true, - 33 => has_sps = true, - 34 => has_pps = true, - 19 | 20 => has_idr = true, - _ => {} - } - } - - Ok(has_vps && has_sps && has_pps && has_idr) - } - EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { - Err(CaptureError::UnsupportedCodec(codec)) - } - } -} - impl From for VideoCodec { fn from(value: EncodedVideoCodec) -> Self { match value { @@ -233,85 +197,3 @@ const _: () = { fn _assert_object_safe(_: &dyn EncodedVideoSource) {} }; -pub(crate) fn h264_nal_type(nal: &[u8]) -> Result { - let header = nal.first().ok_or(CaptureError::EmptyPayload)?; - Ok(header & 0x1f) -} - -pub(crate) fn h265_nal_type(nal: &[u8]) -> Result { - if nal.is_empty() { - return Err(CaptureError::EmptyPayload); - } - if nal.len() < 2 { - return Err(CaptureError::H265NalTooShort); - } - Ok((nal[0] >> 1) & 0x3f) -} - -pub(crate) fn annex_b_payload(nal_units: &[&[u8]]) -> Result, CaptureError> { - if nal_units.is_empty() { - return Err(CaptureError::EmptyPayload); - } - let len = nal_units.iter().try_fold(0usize, |len, nal| { - if nal.is_empty() { - Err(CaptureError::EmptyPayload) - } else { - Ok(len + ANNEX_B_START_CODE.len() + nal.len()) - } - })?; - - let mut payload = Vec::with_capacity(len); - for nal in nal_units { - payload.extend_from_slice(&ANNEX_B_START_CODE); - payload.extend_from_slice(nal); - } - Ok(payload) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn h264_keyframe_requires_idr_nal() { - let sps = [0x67, 1, 2, 3]; - let idr = [0x65, 4, 5, 6]; - let non_idr = [0x61, 1, 2]; - - assert!(is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &idr]).unwrap()); - assert!(!is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &non_idr]).unwrap()); - } - - #[test] - fn h265_keyframe_requires_parameter_sets_and_idr() { - let vps = [0x40, 1, 2]; - let sps = [0x42, 1, 2]; - let pps = [0x44, 1, 2]; - let idr_w_radl = [19 << 1, 1, 3]; - let cra = [21 << 1, 1, 3]; - - assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &idr_w_radl]).unwrap()); - assert!( - is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &idr_w_radl]).unwrap() - ); - assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &cra]).unwrap()); - } - - #[test] - fn h265_rejects_too_short_nal_header() { - let err = is_keyframe_nalus(EncodedVideoCodec::H265, &[&[0x26]]).unwrap_err(); - assert_eq!(err, CaptureError::H265NalTooShort); - } - - #[test] - fn annex_b_payload_prefixes_each_nal_unit() { - let payload = annex_b_payload(&[&[0x67, 1, 2, 3], &[0x65, 4, 5, 6]]).unwrap(); - assert_eq!(payload, vec![0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5, 6]); - } - - #[test] - fn annex_b_payload_rejects_empty_input() { - assert_eq!(annex_b_payload(&[]).unwrap_err(), CaptureError::EmptyPayload); - assert_eq!(annex_b_payload(&[&[]]).unwrap_err(), CaptureError::EmptyPayload); - } -} diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs index 95e4d77be..c394a7b95 100644 --- a/livekit-capture/src/encoded/pump.rs +++ b/livekit-capture/src/encoded/pump.rs @@ -17,7 +17,7 @@ use crate::{ encoded::{EncodedFrameType, EncodedVideoSource, OwnedEncodedAccessUnit}, - error::CaptureError, + error::SourceError, pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, }; use livekit::{ @@ -161,9 +161,13 @@ fn capture_access_unit( rtc_source: &NativeVideoSource, access_unit: &OwnedEncodedAccessUnit, frame_metadata: Option, -) -> Result<(), CaptureError> { +) -> Result<(), PumpError> { + // An empty payload is a violation of the source contract, so it is + // attributed to the source rather than the pump. if access_unit.payload.is_empty() { - return Err(CaptureError::EmptyPayload); + return Err(PumpError::Source(SourceError::new( + "source produced an access unit with an empty payload", + ))); } let frame = EncodedVideoFrame { @@ -174,7 +178,7 @@ fn capture_access_unit( resolution: access_unit.resolution.into(), frame_metadata, }; - rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(CaptureError::CaptureFailed) + rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(PumpError::CaptureFailed) } #[cfg(test)] @@ -274,8 +278,9 @@ mod tests { let mut unit = access_unit(1, EncodedFrameType::Key); unit.payload = Bytes::new(); - let result = EncodedVideoPump::new(FakeEncodedSource::new([unit])).run(); - assert!(matches!(result, Err(PumpError::Capture(CaptureError::EmptyPayload)))); + let error = EncodedVideoPump::new(FakeEncodedSource::new([unit])).run().unwrap_err(); + assert!(matches!(&error, PumpError::Source(_))); + assert!(error.to_string().contains("empty payload")); } #[test] diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index 8c7a4621a..60c5d9ce6 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -12,11 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Error types shared across capture paths. +//! The type-erased error shared by capture sources. +//! +//! Concrete error types live with what produces them: each source module +//! defines its own error, the parsing helpers define +//! [`H26xParseError`](crate::encoded::h26x::H26xParseError), and the pumps +//! report through [`PumpError`](crate::pump::PumpError). -use crate::encoded::EncodedVideoCodec; use std::{error::Error as StdError, fmt}; -use thiserror::Error; /// Error returned by a capture source. /// @@ -44,23 +47,3 @@ impl StdError for SourceError { self.0.source() } } - -/// Error returned by capture helpers. -#[derive(Debug, Error, PartialEq, Eq)] -pub enum CaptureError { - /// Encoded payload is empty. - #[error("encoded payload is empty")] - EmptyPayload, - /// H.265 NAL unit is too short to contain its header. - #[error("H.265 NAL unit is too short")] - H265NalTooShort, - /// Codec is represented by the API but not yet supported by native passthrough. - #[error("encoded passthrough does not support {0:?} yet")] - UnsupportedCodec(EncodedVideoCodec), - /// Encoded payload or transport data is malformed. - #[error("invalid encoded data: {0}")] - InvalidEncodedData(&'static str), - /// The underlying source rejected the frame. - #[error("capture source rejected the frame")] - CaptureFailed, -} diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index a4fa235e2..9275905e0 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -23,7 +23,7 @@ //! spawn into the same [`RunningPump`] defined here, so running pumps of //! either kind are supervised uniformly. -use crate::error::{CaptureError, SourceError}; +use crate::error::SourceError; use std::{ any::Any, io, @@ -39,12 +39,12 @@ use thiserror::Error; /// Error returned by a pump run. #[derive(Debug, Error)] pub enum PumpError { - /// The capture source failed. + /// The capture source failed or violated its contract. #[error("capture source failed: {0}")] Source(#[from] SourceError), /// The RTC source rejected a frame. - #[error("frame capture failed: {0}")] - Capture(#[from] CaptureError), + #[error("capture source rejected the frame")] + CaptureFailed, /// The pump thread panicked. #[error("pump panicked: {0}")] Panicked(String), diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index f51c60e0d..76544ff15 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -21,10 +21,10 @@ use thiserror::Error; use crate::{ encoded::{ - h26x::{access_unit_from_annex_b, access_unit_from_h264_avc}, + h26x::{access_unit_from_annex_b, access_unit_from_h264_avc, H26xParseError}, EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, OwnedEncodedAccessUnit, }, - error::{CaptureError, SourceError}, + error::SourceError, primitive::VideoResolution, pump::PumpStop, }; @@ -435,7 +435,7 @@ impl GStreamerVideoSource { frame_type, self.resolution, ) - .map_err(GStreamerVideoSourceError::Capture) + .map_err(GStreamerVideoSourceError::Parse) } fn timestamp_us(&mut self, buffer: &gst::BufferRef) -> i64 { @@ -613,9 +613,9 @@ pub enum GStreamerVideoSourceError { /// The sample buffer could not be mapped for reading. #[error("failed to map GStreamer buffer for reading: {0}")] MapReadable(String), - /// Access-unit construction failed. + /// Access-unit parsing failed. #[error(transparent)] - Capture(CaptureError), + Parse(H26xParseError), } fn access_unit_from_sample_payload( @@ -624,7 +624,7 @@ fn access_unit_from_sample_payload( timestamp_us: i64, frame_type: EncodedFrameType, resolution: VideoResolution, -) -> Result { +) -> Result { match sample_format { GStreamerSampleFormat::H264AnnexB => access_unit_from_annex_b( EncodedVideoCodec::H264, @@ -643,7 +643,7 @@ fn access_unit_from_sample_payload( ), GStreamerSampleFormat::AccessUnit { codec } => { if payload.is_empty() { - return Err(CaptureError::EmptyPayload); + return Err(H26xParseError::EmptyPayload); } Ok(OwnedEncodedAccessUnit::new( From 5ada11d6e836202df0fb3f010147b19ef93800ed Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:04:39 -0700 Subject: [PATCH 48/56] Prefix source features --- livekit-capture/AGENTS.md | 4 +++- livekit-capture/Cargo.toml | 8 ++++---- livekit-capture/README.md | 16 ++++++++-------- livekit-capture/src/sources/mod.rs | 6 +++--- livekit-ffi/Cargo.toml | 6 +++--- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/livekit-capture/AGENTS.md b/livekit-capture/AGENTS.md index d2d247b85..19fa12189 100644 --- a/livekit-capture/AGENTS.md +++ b/livekit-capture/AGENTS.md @@ -3,10 +3,12 @@ ## Adding a source - Add a feature gate in `Cargo.toml` + - Name the feature after the source module with a `source-` prefix + (e.g., module `gstreamer` → feature `source-gstreamer`) - If the new source requires dependencies, they should only be include when feature is enabled - Define a module for the new source in `src/sources/mod.rs`: ```rust - #[cfg(feature = "my-source")] + #[cfg(feature = "source-my-source")] pub mod my_source; ``` - Follow conventions for existing sources diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 4c517af84..9cc4895f5 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -23,7 +23,7 @@ yuv-sys = { workspace = true, features = ["jpeg"], optional = true } tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] -default = ["demo", "device", "gstreamer"] # TODO: Remove after testing +default = ["source-demo", "source-device", "source-gstreamer"] # TODO: Remove after testing serde = ["dep:serde"] schemars = ["dep:schemars", "serde"] @@ -32,8 +32,8 @@ schemars = ["dep:schemars", "serde"] tokio = ["tokio/rt"] # Pixel sources -demo = [] -device = [ +source-demo = [] +source-device = [ "dep:yuv-sys", # macOS backend "dep:dispatch2", @@ -76,7 +76,7 @@ device = [ ] # Encoded sources -gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] +source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] [target.'cfg(target_os = "macos")'.dependencies] dispatch2 = { version = "0.3.1", default-features = false, features = ["std"], optional = true } diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 51f77f81b..e4f42832d 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -46,11 +46,11 @@ let stats = running.stop_and_join_async().await?; ## Sources -Each source lives in its own module under `sources`, behind the Cargo feature -of the same name. Its module documents it. - -| Feature | Source | Path | -| ----------- | ---------------------- | ------- | -| `demo` | `DemoVideoSource` | pixel | -| `device` | `DeviceVideoSource` | pixel | -| `gstreamer` | `GStreamerVideoSource` | encoded | +Each source lives in its own module under `sources`, behind a Cargo feature +named `source-`. Its module documents it. + +| Feature | Source | Path | +| ------------------ | ---------------------- | ------- | +| `source-demo` | `DemoVideoSource` | pixel | +| `source-device` | `DeviceVideoSource` | pixel | +| `source-gstreamer` | `GStreamerVideoSource` | encoded | diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index f743c92d6..f775b69ae 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "demo")] +#[cfg(feature = "source-demo")] pub mod demo; -#[cfg(feature = "device")] +#[cfg(feature = "source-device")] pub mod device; -#[cfg(feature = "gstreamer")] +#[cfg(feature = "source-gstreamer")] pub mod gstreamer; diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index abaa12005..6ba158fc3 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -24,9 +24,9 @@ tracing = ["tokio/tracing", "console-subscriber"] # GStreamer. capture = [ "dep:livekit-capture", - "livekit-capture/demo", - "livekit-capture/device", - "livekit-capture/gstreamer", + "livekit-capture/source-demo", + "livekit-capture/source-device", + "livekit-capture/source-gstreamer", "livekit-capture/tokio", ] From 88cc7fa95c18f0a129e4266f0f717ec1ec78d232 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:35:04 -0700 Subject: [PATCH 49/56] Refine documentation --- livekit-capture/README.md | 50 ++++++----- livekit-capture/src/encoded/h26x.rs | 33 +++++--- livekit-capture/src/encoded/mod.rs | 46 +++++----- livekit-capture/src/encoded/pump.rs | 34 ++++---- livekit-capture/src/error.rs | 14 ++-- livekit-capture/src/lib.rs | 7 ++ livekit-capture/src/pixel/mod.rs | 34 ++++---- livekit-capture/src/pixel/pump.rs | 36 ++++---- livekit-capture/src/primitive.rs | 12 +-- livekit-capture/src/pump.rs | 45 +++++----- livekit-capture/src/sources/demo.rs | 17 ++-- .../src/sources/device/avfoundation.rs | 10 +-- livekit-capture/src/sources/device/mod.rs | 72 ++++++++-------- .../src/sources/device/timestamp.rs | 4 +- livekit-capture/src/sources/device/v4l2.rs | 10 +-- livekit-capture/src/sources/gstreamer.rs | 84 ++++++++++--------- livekit-capture/src/sources/mod.rs | 3 + livekit-capture/src/utils.rs | 5 +- 18 files changed, 260 insertions(+), 256 deletions(-) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index e4f42832d..06e6c2f3b 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -1,37 +1,39 @@ # LiveKit Capture -Video capture sources, and the machinery that publishes them with the LiveKit -[Rust SDK](../livekit/README.md). A capture backend implements one small trait. An application then -runs and supervises every backend the same way. +This crate provides video capture sources and the pumps that publish them +with the LiveKit [Rust SDK](../livekit/README.md). Pick a ready-made source, +or implement one small trait to add your own. The application runs and +supervises every source the same way. -## Source, pump, running pump +## Source and pump -Three concepts make up the crate. Video reaches a LiveKit track in one of two +Two concepts make up the crate. Video reaches a LiveKit track in one of two forms, so the source and the pump each have two variants. -**A source** produces frames or access units, one blocking call at a time. It -is the only trait a backend implements. A `pixel::PixelVideoSource` produces -libwebrtc `VideoFrame`s from a device such as a camera, and the WebRTC encoder -encodes them. An `encoded::EncodedVideoSource` produces access units from a -producer that encoded them already, such as an encoding pipeline. Passthrough -sends those to the wire with no re-encode. +**A source** produces video, one blocking call at a time. Use a ready-made +source (see [Sources](#sources)), or implement the trait to add your own: -**A pump** bridges one source into a publishable RTC track: +- A `pixel::PixelVideoSource` produces raw `VideoFrame`s — from a camera, for + example. The WebRTC encoder encodes them. +- An `encoded::EncodedVideoSource` produces access units that are already + encoded — from an encoding pipeline, for example. The SDK sends them to the + wire without re-encoding (passthrough). When frames are encoded upstream, + this removes an extra decode and encode step and lowers latency. + +**A pump** connects one source to an RTC video source: `pixel::PixelVideoPump` or `encoded::EncodedVideoPump`. It builds the matching RTC video source, derives the publish options, and runs the capture -loop. - -**A running pump** is a pump on a dedicated thread. Both pump kinds spawn into -the same `pump::RunningPump`, so an application supervises pumps of either kind -the same way. +loop. Spawn a pump onto a dedicated thread and it becomes a +`pump::RunningPump`. Both pump kinds spawn into the same type, so an +application supervises them the same way. Stop a running pump from any +thread through its stop handle. -Sources block, so the pumps are synchronous code on plain threads. Only pump -construction needs the context of the async runtime that drives the SDK. +Sources block, so the pumps run synchronous code on plain threads. ## Publishing a track -A pump supplies both pieces that the SDK needs, so publication is the same for -either path. +A pump supplies the RTC source and the publish options, so publication is the +same for either path. ```rust let pump = PixelVideoPump::new(DemoVideoSource::new(config)?); @@ -41,15 +43,17 @@ let options = pump.publish_options(); room.local_participant().publish_track(LocalTrack::Video(track), options).await?; let running = pump.spawn()?; + +// On shutdown: let stats = running.stop_and_join_async().await?; ``` ## Sources Each source lives in its own module under `sources`, behind a Cargo feature -named `source-`. Its module documents it. +named `source-`. Each module documents its source. -| Feature | Source | Path | +| Feature | Source | Kind | | ------------------ | ---------------------- | ------- | | `source-demo` | `DemoVideoSource` | pixel | | `source-device` | `DeviceVideoSource` | pixel | diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index bc3d65489..a14b86710 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! H.264/H.265 parsing helpers: NAL-unit splitting, access-unit assembly and -//! delimiting, and keyframe detection for the encoded ingest paths. +//! H.264/H.265 parsing helpers: NAL-unit splitting, access-unit assembly, +//! and keyframe detection. use crate::{ encoded::{EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit}, @@ -65,15 +65,19 @@ pub(crate) trait AccessUnitParser { fn flush(&mut self) -> Result, H26xParseError>; } -/// H26x Annex-B parser state. +/// Incremental parser that splits an H.264 or H.265 Annex-B byte stream +/// into access units. +/// +/// Call [`AnnexBAccessUnitParser::push`] as bytes arrive, and +/// [`AnnexBAccessUnitParser::flush`] at the end of the stream. #[derive(Debug, Clone)] pub struct AnnexBAccessUnitParser { codec: EncodedVideoCodec, pending: Vec, - /// NAL ranges found in `pending`; the last range's end is provisional - /// until the next start code (or flush) confirms it. + // NAL ranges found in `pending`; the last range's end is provisional + // until the next start code (or flush) confirms it. nal_ranges: Vec>, - /// Offset up to which `pending` has been scanned for start codes. + // Offset up to which `pending` has been scanned for start codes. scan_cursor: usize, next_timestamp_us: i64, frame_interval_us: i64, @@ -96,7 +100,10 @@ pub(crate) struct AvcAccessUnitParser { } impl AnnexBAccessUnitParser { - /// Creates a parser for H.264 or H.265 Annex-B byte streams. + /// Creates a parser for an H.264 or H.265 Annex-B byte stream. + /// + /// Access units get timestamps that start at `start_timestamp_us` and + /// step by `frame_interval_us`. pub fn new( codec: EncodedVideoCodec, start_timestamp_us: i64, @@ -121,13 +128,17 @@ impl AnnexBAccessUnitParser { }) } - /// Pushes encoded bytes and returns the next complete access unit if one is found. + /// Pushes encoded bytes and returns the next complete access unit, if + /// one is found. + /// + /// One call returns at most one access unit. Push an empty slice to + /// pull further access units that are already buffered. pub fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError> { self.pending.extend_from_slice(bytes); self.drain_next(false) } - /// Flushes the pending bytes as the final access unit. + /// Flushes the remaining buffered bytes as the final access unit. pub fn flush(&mut self) -> Result, H26xParseError> { self.drain_next(true) } @@ -423,7 +434,9 @@ pub fn access_unit_from_nalus( access_unit_from_annex_b(codec, payload, timestamp_us, resolution) } -/// Returns true when an Annex-B access unit contains an intra/key picture. +/// Returns `true` when an Annex-B access unit is a key frame: an IDR +/// picture for H.264, or parameter sets (VPS/SPS/PPS) plus an IDR picture +/// for H.265. pub fn is_keyframe_annex_b(codec: EncodedVideoCodec, bytes: &[u8]) -> Result { let nals = annex_b_nalus(bytes); is_keyframe_nalus(codec, &nals) diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index 5f31e3c23..872865c2d 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -12,14 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Encoded video: codec vocabulary, access units, the source contract for -//! pre-encoded ingest, and the pump. +//! Pre-encoded video: the source trait, access units, and the pump. //! -//! Sources produce crate-owned access units — the vocabulary the parsing -//! and validation helpers speak — and [`EncodedVideoPump`] bridges them into -//! an RTC track as passthrough. The source trait is object-safe and -//! implemented for `Box`, so sources can be constructed dynamically -//! and driven through the same generic pump. +//! A source produces [`OwnedEncodedAccessUnit`]s, and [`EncodedVideoPump`] +//! publishes them to an RTC track as passthrough — without re-encoding. +//! +//! [`EncodedVideoSource`] is object-safe and implemented for `Box`, +//! so sources constructed dynamically run through the same generic pump. use crate::{error::SourceError, primitive::VideoResolution, pump::PumpStop}; use bytes::Bytes; @@ -42,34 +41,35 @@ pub trait EncodedVideoSource: Send { /// Nominal output resolution, used to size the RTC source. fn resolution(&self) -> VideoResolution; - /// Codec produced by this source; fixed for the source's lifetime. + /// Codec produced by this source. The codec is fixed for the source's + /// lifetime. fn codec(&self) -> EncodedVideoCodec; - /// Blocks until the next access unit is available, returning `Ok(None)` - /// when the source reaches the end of its stream. + /// Blocks until the next access unit is available. Returns `Ok(None)` + /// at the end of the stream. /// - /// Sources must return promptly (with `Ok(None)`) once `stop` fires: - /// integrate it into the blocking wait, or bound each wait so the token - /// is observed within a frame interval or so. The pump distinguishes a - /// stop from end of stream via the token. + /// Implementations must return `Ok(None)` promptly once `stop` fires: + /// integrate the token into the blocking wait, or bound each wait to + /// about one frame interval. The pump uses the token to tell a stop + /// from the end of the stream. /// - /// Access units must carry a non-empty payload; the pump reports a - /// violation as a source error. + /// Access units must carry a non-empty payload. The pump reports an + /// empty payload as a source error. fn next_access_unit( &mut self, stop: &PumpStop, ) -> Result, SourceError>; /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to - /// the producer so it can emit an IDR. + /// the producer so it can emit a keyframe. /// - /// The default implementation does nothing, for transports that cannot + /// The default implementation does nothing, for sources that cannot /// influence the upstream encoder. fn request_keyframe(&mut self) {} /// Forwards a downstream rate-control target to the producer. /// - /// The default implementation does nothing, for transports that cannot + /// The default implementation does nothing, for sources that cannot /// influence the upstream encoder. fn update_rate_control(&mut self, _target: EncodedRateControl) {} } @@ -108,20 +108,20 @@ pub enum EncodedFrameType { /// Owned encoded video access unit. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OwnedEncodedAccessUnit { - /// Encoded codec. + /// Codec of the payload. pub codec: EncodedVideoCodec, /// Encoded payload bytes. pub payload: Bytes, /// Capture timestamp in microseconds. pub timestamp_us: i64, - /// Encoded frame type. + /// Frame type. pub frame_type: EncodedFrameType, - /// Encoded frame resolution in pixels. + /// Frame resolution in pixels. pub resolution: VideoResolution, } impl OwnedEncodedAccessUnit { - /// Creates an owned encoded access unit from contiguous bytes. + /// Creates an access unit. pub fn new( codec: EncodedVideoCodec, payload: impl Into, diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs index c394a7b95..a7617b070 100644 --- a/livekit-capture/src/encoded/pump.rs +++ b/livekit-capture/src/encoded/pump.rs @@ -32,13 +32,13 @@ use std::{fmt, io}; /// Callback that supplies packet-trailer metadata for an access unit. type FrameMetadataFn = Box Option + Send>; -/// Pumps an [`EncodedVideoSource`] into an RTC video source, publishing -/// access units as passthrough. +/// Pumps an [`EncodedVideoSource`] into an RTC video source as passthrough, +/// without re-encoding. /// -/// Downstream keyframe requests (PLI/FIR, late subscriber) and rate-control -/// targets are polled between access units and forwarded to the source. -/// Pre-roll delta frames are dropped until the first keyframe, since -/// decoding can only start at a keyframe. +/// Downstream keyframe requests and rate-control targets are forwarded to +/// the source between access units. Delta frames that arrive before the +/// first keyframe are dropped, because decoding can only start at a +/// keyframe. pub struct EncodedVideoPump { source: S, rtc_source: NativeVideoSource, @@ -47,7 +47,7 @@ pub struct EncodedVideoPump { } impl EncodedVideoPump { - /// Creates a pump for an encoded source, building the matching RTC + /// Creates a pump for an encoded source and builds the matching RTC /// source. pub fn new(source: S) -> Self { let rtc_source = NativeVideoSource::new_encoded(source.resolution().into()); @@ -55,11 +55,11 @@ impl EncodedVideoPump { } /// Sets a callback that supplies packet-trailer metadata for each access - /// unit before it is captured. + /// unit. /// - /// Metadata is only propagated to subscribers when the corresponding - /// [`TrackPublishOptions::frame_metadata_features`] are enabled before - /// publishing the local track. + /// Subscribers receive metadata only when the matching + /// [`TrackPublishOptions::frame_metadata_features`] are enabled on the + /// published track. pub fn with_frame_metadata( mut self, frame_metadata: impl FnMut(&OwnedEncodedAccessUnit) -> Option + Send + 'static, @@ -93,11 +93,11 @@ impl EncodedVideoPump { &self.source } - /// Runs the pump on the calling thread until the source ends, a failure, - /// or the stop handle fires. + /// Runs the pump on the calling thread until the source ends, an error + /// occurs, or the stop handle fires. /// - /// Sources block, so callers on an async runtime should run this on a - /// dedicated thread (see [`EncodedVideoPump::spawn`]) or a blocking pool. + /// Sources block. On an async runtime, run this on a dedicated thread + /// (see [`EncodedVideoPump::spawn`]) or a blocking pool. pub fn run(mut self) -> Result { let mut frames_captured = 0; let mut awaiting_initial_keyframe = true; @@ -137,8 +137,8 @@ impl EncodedVideoPump { /// Runs the pump on a dedicated thread. /// - /// Panics on the pump thread are caught and reported as - /// [`PumpError::Panicked`] when the pump is joined. + /// A panic on the pump thread is reported as [`PumpError::Panicked`] + /// when the pump is joined. pub fn spawn(self) -> io::Result where S: 'static, diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index 60c5d9ce6..be0991726 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -12,20 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! The type-erased error shared by capture sources. +//! The error type shared by capture sources. //! -//! Concrete error types live with what produces them: each source module -//! defines its own error, the parsing helpers define -//! [`H26xParseError`](crate::encoded::h26x::H26xParseError), and the pumps -//! report through [`PumpError`](crate::pump::PumpError). +//! Concrete errors live with what produces them: each source module defines +//! its own error type, and the pumps report through +//! [`PumpError`](crate::pump::PumpError). use std::{error::Error as StdError, fmt}; /// Error returned by a capture source. /// -/// Backend-specific errors are type-erased so sources stay usable as trait -/// objects; the wrapped error remains reachable for display and through -/// [`StdError::source`]. +/// `Display` and [`StdError::source`] delegate to the wrapped backend +/// error. #[derive(Debug)] pub struct SourceError(Box); diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index 408c87fdc..c9c5b526f 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -12,6 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Video capture for the LiveKit Rust SDK. +//! +//! A capture source produces video: pixel frames ([`pixel`]) or pre-encoded +//! access units ([`encoded`]). A pump drives a source and publishes its +//! output to an RTC video source. Ready-made sources live in [`sources`] +//! and can be enabled by their corresponding features. + pub mod encoded; pub mod error; pub mod pixel; diff --git a/livekit-capture/src/pixel/mod.rs b/livekit-capture/src/pixel/mod.rs index cb7f1cf82..8979e5c0e 100644 --- a/livekit-capture/src/pixel/mod.rs +++ b/livekit-capture/src/pixel/mod.rs @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Pixel (unencoded) video: the source contract and the pump. +//! Pixel (unencoded) video: the source trait and its pump. //! -//! Sources yield libwebrtc [`VideoFrame`](livekit::webrtc::video_frame::VideoFrame)s -//! directly, so any [`VideoBuffer`](livekit::webrtc::video_frame::VideoBuffer) -//! implementation — CPU planes or platform-native — reaches the RTC track -//! without an intermediate copy. [`PixelVideoPump`] drives a source and -//! publishes its frames through the WebRTC encoder. The source trait is -//! object-safe and implemented for `Box`, so sources can be -//! constructed dynamically and driven through the same generic pump. +//! A source yields libwebrtc [`VideoFrame`](livekit::webrtc::video_frame::VideoFrame)s, +//! so any [`VideoBuffer`](livekit::webrtc::video_frame::VideoBuffer) — CPU +//! planes or platform-native — passes to the RTC source without an +//! intermediate copy. [`PixelVideoPump`] drives a source and publishes its +//! frames through the WebRTC encoder. +//! +//! [`PixelVideoSource`] is object-safe and implemented for `Box`, +//! so sources constructed dynamically run through the same generic pump. mod pump; @@ -35,16 +36,17 @@ pub trait PixelVideoSource: Send { /// Nominal output resolution, used to size the RTC source. fn resolution(&self) -> VideoResolution; - /// Blocks until the next frame is available, returning `Ok(None)` when - /// the source reaches the end of its stream. + /// Blocks until the next frame is available. Returns `Ok(None)` at the + /// end of the stream. /// - /// Sources must return promptly (with `Ok(None)`) once `stop` fires: - /// integrate it into the blocking wait, or bound each wait so the token - /// is observed within a frame interval or so. The pump distinguishes a - /// stop from end of stream via the token. + /// Implementations must return `Ok(None)` promptly once `stop` fires: + /// integrate the token into the blocking wait, or bound each wait to + /// about one frame interval. The pump uses the token to tell a stop + /// from the end of the stream. /// - /// Sources may pre-fill the frame's `frame_metadata`; a metadata - /// callback set on the pump takes precedence when it returns `Some`. + /// Implementations can pre-fill the frame's `frame_metadata`. A + /// metadata callback set on the pump takes precedence when it returns + /// `Some`. fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError>; } diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs index 58487db31..97500b1e2 100644 --- a/livekit-capture/src/pixel/pump.rs +++ b/livekit-capture/src/pixel/pump.rs @@ -30,8 +30,8 @@ use std::{fmt, io}; /// Callback that supplies packet-trailer metadata for a pixel frame. type FrameMetadataFn = Box Option + Send>; -/// Pumps a [`PixelVideoSource`] into an RTC video source, publishing frames -/// through the WebRTC encoder. +/// Pumps a [`PixelVideoSource`] into an RTC video source and publishes its +/// frames through the WebRTC encoder. pub struct PixelVideoPump { source: S, rtc_source: NativeVideoSource, @@ -40,24 +40,18 @@ pub struct PixelVideoPump { } impl PixelVideoPump { - /// Creates a pump for a pixel source, building the matching RTC source. - /// - /// This must be called from the context of the async runtime driving the - /// SDK, because the RTC source spawns its keepalive task at construction. - /// The pump itself runs on plain threads. + /// Creates a pump for a pixel source and builds the matching RTC source. pub fn new(source: S) -> Self { let rtc_source = NativeVideoSource::new(source.resolution().into(), false); Self { source, rtc_source, stop: PumpStop::new(), frame_metadata: None } } - /// Sets a callback that supplies packet-trailer metadata for each frame - /// before it is captured. + /// Sets a callback that supplies packet-trailer metadata for each frame. /// - /// When the callback returns `Some`, it overrides any metadata the - /// source pre-filled on the frame. Metadata is only propagated to - /// subscribers when the corresponding - /// [`TrackPublishOptions::frame_metadata_features`] are enabled before - /// publishing the local track. + /// When the callback returns `Some`, it overrides metadata the source + /// pre-filled on the frame. Subscribers receive metadata only when the + /// matching [`TrackPublishOptions::frame_metadata_features`] are enabled + /// on the published track. pub fn with_frame_metadata( mut self, frame_metadata: impl FnMut(&BoxVideoFrame) -> Option + Send + 'static, @@ -71,7 +65,7 @@ impl PixelVideoPump { RtcVideoSource::Native(self.rtc_source.clone()) } - /// Returns publish options appropriate for a pixel source. + /// Returns publish options for a pixel source. pub fn publish_options(&self) -> TrackPublishOptions { TrackPublishOptions::default() } @@ -86,11 +80,11 @@ impl PixelVideoPump { &self.source } - /// Runs the pump on the calling thread until the source ends, a failure, - /// or the stop handle fires. + /// Runs the pump on the calling thread until the source ends, an error + /// occurs, or the stop handle fires. /// - /// Sources block, so callers on an async runtime should run this on a - /// dedicated thread (see [`PixelVideoPump::spawn`]) or a blocking pool. + /// Sources block. On an async runtime, run this on a dedicated thread + /// (see [`PixelVideoPump::spawn`]) or a blocking pool. pub fn run(mut self) -> Result { let mut frames_captured = 0; let exit = loop { @@ -119,8 +113,8 @@ impl PixelVideoPump { /// Runs the pump on a dedicated thread. /// - /// Panics on the pump thread are caught and reported as - /// [`PumpError::Panicked`] when the pump is joined. + /// A panic on the pump thread is reported as [`PumpError::Panicked`] + /// when the pump is joined. pub fn spawn(self) -> io::Result where S: 'static, diff --git a/livekit-capture/src/primitive.rs b/livekit-capture/src/primitive.rs index 60251c421..4ed757b09 100644 --- a/livekit-capture/src/primitive.rs +++ b/livekit-capture/src/primitive.rs @@ -12,11 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Domain-neutral video primitives shared across capture paths and backends. -//! -//! These types carry no capture- or codec-specific semantics, so they can serve -//! as a common vocabulary for frame geometry and related quantities across -//! crates. +//! Basic video types, such as [`VideoResolution`]. // TODO: in a future refactor, move these types into their own // crate (e.g., `livekit-video-primitives`) so all crates in this workspace can work @@ -50,16 +46,14 @@ impl VideoResolution { Self { width, height } } - /// Returns the ratio between the width and height components. - /// - /// If the height component is zero, the result is `None`. + /// Returns the width divided by the height, or `None` if the height is + /// zero. /// /// ``` /// # use livekit_capture::primitive::VideoResolution; /// assert_eq!(VideoResolution::new(1920, 960).aspect_ratio(), Some(2.0)); /// assert_eq!(VideoResolution::new(1920, 0).aspect_ratio(), None); /// ``` - /// pub fn aspect_ratio(&self) -> Option { if self.height == 0 { return None; diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs index 9275905e0..462f9ef65 100644 --- a/livekit-capture/src/pump.rs +++ b/livekit-capture/src/pump.rs @@ -12,16 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Machinery shared by the capture pumps. +//! Types shared by the capture pumps. //! -//! The kind-specific pumps live with their kinds — -//! [`PixelVideoPump`](crate::pixel::PixelVideoPump) and -//! [`EncodedVideoPump`](crate::encoded::EncodedVideoPump) — and are generic -//! over a concrete source, so statically-known sources pay for no type -//! erasure. Applications that construct sources dynamically box them at -//! their edge (`PixelVideoPump>`). Both pumps -//! spawn into the same [`RunningPump`] defined here, so running pumps of -//! either kind are supervised uniformly. +//! The pumps live with their frame kinds: +//! [`PixelVideoPump`](crate::pixel::PixelVideoPump) pumps pixel frames, and +//! [`EncodedVideoPump`](crate::encoded::EncodedVideoPump) pumps encoded +//! access units. Both spawn into the same [`RunningPump`]. use crate::error::SourceError; use std::{ @@ -53,7 +49,7 @@ pub enum PumpError { /// Why a pump run ended successfully. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PumpExit { - /// The stop handle was fired. + /// The stop handle fired. Stopped, /// The source reached the end of its stream. EndOfStream, @@ -71,13 +67,13 @@ pub struct PumpStats { /// Cancellation handle for a pump. /// -/// Cheap to clone; wire it to a shutdown signal and call [`PumpStop::stop`] -/// from any thread to make the pump return after the frame in flight. +/// The handle is cheap to clone. Call [`PumpStop::stop`] from any thread to +/// make the pump return after the frame in flight. #[derive(Debug, Clone, Default)] pub struct PumpStop(Arc); impl PumpStop { - /// Creates an un-stopped handle. + /// Creates a new handle. pub fn new() -> Self { Self::default() } @@ -87,14 +83,14 @@ impl PumpStop { self.0.store(true, Ordering::Release); } - /// Returns true once [`PumpStop::stop`] has been called. + /// Returns whether [`PumpStop::stop`] was called. pub fn is_stopped(&self) -> bool { self.0.load(Ordering::Acquire) } } -/// Spawns a pump run on a dedicated thread, wiring panic capture and the -/// completion signal shared by both pump kinds. +/// Spawns a pump run on a dedicated thread with panic capture and a +/// completion signal. pub(crate) fn spawn_pump( stop: PumpStop, run: impl FnOnce() -> Result + Send + 'static, @@ -122,15 +118,15 @@ fn panic_message(panic: &(dyn Any + Send)) -> String { } } -/// A pump of either kind running on a dedicated thread. +/// A pump of either kind that runs on a dedicated thread. /// -/// Stopping takes effect between frames: a source blocked waiting for its -/// next frame finishes that wait before the pump observes the signal. +/// A stop takes effect between frames: a source that is blocked on its next +/// frame completes that wait before the pump observes the signal. #[derive(Debug)] pub struct RunningPump { stop: PumpStop, thread: thread::JoinHandle>, - /// Flipped to true by the pump thread just before it exits. + // Flipped to true by the pump thread just before it exits. finished: tokio::sync::watch::Receiver, } @@ -145,7 +141,7 @@ impl RunningPump { self.stop.stop(); } - /// Returns true once the pump thread has exited. + /// Returns whether the pump thread exited. pub fn is_finished(&self) -> bool { self.thread.is_finished() } @@ -165,10 +161,9 @@ impl RunningPump { /// Waits for the pump thread to exit without blocking the async runtime. /// - /// This awaits a completion signal rather than parking a thread, so it is - /// safe to hold across long stretches — for example in a `select!` that - /// supervises every running pump — and works under any async runtime, - /// not just tokio. Panics on the pump thread are reported as + /// This works under any async runtime, not only tokio, and is safe to + /// hold across long waits — for example in a `select!` that supervises + /// every running pump. Panics on the pump thread are reported as /// [`PumpError::Panicked`]. pub async fn join_async(mut self) -> Result { // An error means the sender dropped, which also implies the pump diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index 7251ed2c9..4dcec8a23 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -52,24 +52,23 @@ pub struct DemoVideoSourceConfig { pub framerate_fps: u32, } -/// Pixel video source that produces solid-color frames, cycling through a -/// fixed palette. +/// Pixel video source that produces solid-color frames from a fixed +/// palette. /// -/// The source paces itself to the configured frame rate by sleeping and -/// never reaches end of stream; stop the pump driving it instead. It exists -/// to validate capture integration end to end without a device or pipeline -/// dependency. +/// The source sleeps to pace itself to the configured frame rate. It never +/// reaches the end of its stream — stop the pump that drives it instead. #[derive(Debug)] pub struct DemoVideoSource { config: DemoVideoSourceConfig, - /// One `(y, u, v)` sample triple per palette color. + // One `(y, u, v)` sample triple per palette color. colors: [(u8, u8, u8); PALETTE.len()], started: Option, frame_index: u64, } impl DemoVideoSource { - /// Creates a demo source, rejecting an invalid configuration. + /// Creates a demo source. Returns an error for a zero resolution or + /// frame rate. pub fn new(config: DemoVideoSourceConfig) -> Result { let VideoResolution { width, height } = config.resolution; if width == 0 || height == 0 { @@ -88,7 +87,7 @@ impl DemoVideoSource { } } -/// Error returned when a [`DemoVideoSourceConfig`] cannot produce frames. +/// Error returned for an invalid [`DemoVideoSourceConfig`]. #[derive(Debug, Error)] pub enum DemoVideoSourceConfigError { /// The configured resolution has a zero component. diff --git a/livekit-capture/src/sources/device/avfoundation.rs b/livekit-capture/src/sources/device/avfoundation.rs index 76702b968..4fa6eded2 100644 --- a/livekit-capture/src/sources/device/avfoundation.rs +++ b/livekit-capture/src/sources/device/avfoundation.rs @@ -14,10 +14,10 @@ //! macOS device capture backend built on AVFoundation. //! -//! This module is an implementation detail of [`super::DeviceVideoSource`]; +//! This module is an implementation detail of [`super::DeviceVideoSource`]: //! nothing AVFoundation-specific leaves it. Frames are delivered as native -//! IOSurface-backed `CVPixelBuffer`s when the negotiated session supports it -//! (full-range NV12 without software scaling), and converted to I420 +//! IOSurface-backed `CVPixelBuffer`s when the negotiated session supports +//! that (full-range NV12 without software scaling), and converted to I420 //! otherwise. use std::ffi::c_void; @@ -659,8 +659,8 @@ struct QueuedFrame { height: u32, source_format: DeviceFrameFormat, core_video_pixel_format: u32, - /// Wall-clock capture time: the validated sensor timestamp when - /// AVFoundation reports one, the read time otherwise. + // Wall-clock capture time: the validated sensor timestamp when + // AVFoundation reports one, the read time otherwise. capture_wall_time_us: u64, timestamp_us: i64, is_iosurface_backed: bool, diff --git a/livekit-capture/src/sources/device/mod.rs b/livekit-capture/src/sources/device/mod.rs index 616974d40..29173f7b4 100644 --- a/livekit-capture/src/sources/device/mod.rs +++ b/livekit-capture/src/sources/device/mod.rs @@ -14,17 +14,15 @@ //! Camera device capture. //! -//! [`DeviceVideoSource`] is a pixel source that captures frames from a video -//! device attached to the machine, using the platform's native capture stack. -//! The platform integration is an implementation detail: configuration, -//! enumeration ([`devices`]), and errors share one platform-neutral -//! vocabulary, and the same configuration works on every supported platform. -//! On platforms without a capture backend the module still compiles; -//! construction and enumeration fail with +//! [`DeviceVideoSource`] captures pixel frames from a video device through +//! the platform's native capture stack. Configuration, enumeration +//! ([`devices`]), and errors use one platform-neutral vocabulary. On +//! platforms without a backend the module still compiles, and construction +//! and enumeration fail with //! [`DeviceVideoSourceError::UnsupportedPlatform`]. //! //! Where the platform supports it, frames reach the RTC track as -//! platform-native buffers without a CPU copy; otherwise they are converted +//! platform-native buffers without a CPU copy. Otherwise they are converted //! to I420. #[cfg(target_os = "macos")] @@ -65,7 +63,7 @@ pub enum DeviceSelector { Default, /// The device at this position in the platform enumeration order. Index(usize), - /// A platform-stable device identifier, as reported by [`DeviceInfo::id`]. + /// The device with this identifier, as reported by [`DeviceInfo::id`]. Id(String), } @@ -183,8 +181,8 @@ impl fmt::Display for DeviceFormat { /// Format selection requested from a capture device. /// -/// The device negotiates the delivered format; [`DeviceVideoSource::format`] -/// reports the outcome. +/// The device negotiates the delivered format, and +/// [`DeviceVideoSource::format`] reports the outcome. #[derive(Debug, Clone, Default, PartialEq, Eq)] #[cfg_attr( feature = "serde", @@ -232,7 +230,7 @@ pub enum DeviceFormatRequest { )] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct DeviceInfo { - /// Platform-stable device identifier. + /// Device identifier, usable with [`DeviceSelector::Id`]. pub id: String, /// Human-readable device name. pub name: String, @@ -242,7 +240,7 @@ pub struct DeviceInfo { pub manufacturer: Option, /// Capture formats reported by the device. pub formats: Vec, - /// Whether [`DeviceInfo::formats`] is a complete list; some platforms do + /// Whether [`DeviceInfo::formats`] is a complete list. Some platforms do /// not enumerate formats up front. pub formats_complete: bool, } @@ -254,19 +252,18 @@ impl DeviceInfo { } } -/// Lists the video capture devices available on this machine, running the -/// blocking enumeration on the tokio blocking pool. +/// Lists the video capture devices on this machine. /// -/// Requires a running tokio runtime; [`devices_blocking`] is the -/// non-async form. +/// Requires a running tokio runtime: enumeration runs on the tokio blocking +/// pool. Use [`devices_blocking`] outside of async contexts. #[cfg(feature = "tokio")] pub async fn devices() -> Result, SourceError> { crate::utils::run_blocking(devices_blocking).await } -/// Lists the video capture devices available on this machine. +/// Lists the video capture devices on this machine. /// -/// Enumeration queries the platform capture stack and may block briefly. +/// Enumeration queries the platform capture stack and can block briefly. pub fn devices_blocking() -> Result, SourceError> { backend::devices().map_err(SourceError::new) } @@ -288,17 +285,17 @@ pub struct DeviceVideoSourceConfig { pub format: DeviceFormatRequest, } -/// Pixel video source that captures frames from a video device such as a +/// Pixel video source that captures frames from a video device, such as a /// camera. /// /// Construction opens the device and negotiates the capture format, so -/// [`DeviceVideoSource::format`] and the source's nominal resolution are -/// known before the first frame is pumped. Devices never reach end of -/// stream; stop the pump driving the source instead. +/// [`DeviceVideoSource::format`] is known before any frame is pumped. The +/// source never reaches the end of its stream — stop the pump that drives +/// it instead. /// -/// Frames carry a monotonic `timestamp_us`, and each frame's -/// `frame_metadata` is pre-filled with the wall-clock capture time (the -/// device's own capture timestamp when the platform reports a valid one). +/// Frames carry a monotonic `timestamp_us`. Each frame's `frame_metadata` +/// is pre-filled with the wall-clock capture time — the device's own +/// capture timestamp when the platform reports a valid one. pub struct DeviceVideoSource { config: DeviceVideoSourceConfig, format: DeviceFormat, @@ -306,12 +303,11 @@ pub struct DeviceVideoSource { } impl DeviceVideoSource { - /// Creates the source, running blocking device negotiation on the tokio - /// blocking pool. + /// Creates the source. Device negotiation runs on the tokio blocking + /// pool. /// - /// Requires a running tokio runtime. This is the async-constructor - /// convention for capture backends: `new` for async consumers, and - /// [`DeviceVideoSource::new_blocking`] for everything else. + /// Requires a running tokio runtime. Use + /// [`DeviceVideoSource::new_blocking`] outside of async contexts. #[cfg(feature = "tokio")] pub async fn new(config: DeviceVideoSourceConfig) -> Result { crate::utils::run_blocking(move || Self::new_blocking(config)).await @@ -319,11 +315,9 @@ impl DeviceVideoSource { /// Opens the configured device and negotiates the capture format. /// - /// This blocks until the device delivers enough information to establish - /// the format — on some platforms that includes waiting for the first - /// frame, bounded by a timeout. Construction fails loudly on a missing - /// device, an unsatisfiable format request, or a platform without a - /// capture backend. + /// This can block until the device delivers its first frame, bounded by + /// a timeout. Construction fails on a missing device, a format request + /// the device cannot satisfy, or a platform without a capture backend. pub fn new_blocking(config: DeviceVideoSourceConfig) -> Result { let session = backend::Session::open(&config).map_err(SourceError::new)?; let format = session.format(); @@ -337,8 +331,8 @@ impl DeviceVideoSource { /// Returns the negotiated capture format. /// - /// The resolution matches what [`PixelVideoSource::resolution`] reports; - /// the frame format is what the device delivers before any conversion. + /// The resolution matches what [`PixelVideoSource::resolution`] reports. + /// The frame format is what the device delivers before any conversion. pub fn format(&self) -> DeviceFormat { self.format } @@ -403,7 +397,7 @@ pub enum DeviceVideoSourceError { } /// Builds the packet-trailer metadata that device frames are pre-filled -/// with; a metadata callback set on the pump takes precedence. +/// with. A metadata callback set on the pump takes precedence. #[cfg(any(target_os = "macos", target_os = "linux"))] fn capture_frame_metadata( capture_wall_time_us: u64, diff --git a/livekit-capture/src/sources/device/timestamp.rs b/livekit-capture/src/sources/device/timestamp.rs index eb842b435..038e83ce1 100644 --- a/livekit-capture/src/sources/device/timestamp.rs +++ b/livekit-capture/src/sources/device/timestamp.rs @@ -16,8 +16,8 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; -/// Maximum age a backend-reported capture timestamp may have, relative to the -/// wall-clock read time, before it is considered stale and discarded. +/// Maximum age a backend-reported capture timestamp can have, relative to +/// the wall-clock read time, before it is discarded as stale. pub(super) const MAX_CAPTURE_TIMESTAMP_AGE_US: u64 = 5_000_000; /// Returns the current UNIX wall-clock time in microseconds. diff --git a/livekit-capture/src/sources/device/v4l2.rs b/livekit-capture/src/sources/device/v4l2.rs index a636d8976..e718fceb3 100644 --- a/livekit-capture/src/sources/device/v4l2.rs +++ b/livekit-capture/src/sources/device/v4l2.rs @@ -14,10 +14,10 @@ //! Linux device capture backend built on V4L2. //! -//! This module is an implementation detail of [`super::DeviceVideoSource`]; +//! This module is an implementation detail of [`super::DeviceVideoSource`]: //! nothing V4L2-specific leaves it. Frames are converted to I420 on the CPU -//! (via libyuv, with an image-crate fallback for MJPEG streams libyuv -//! rejects). +//! (through libyuv, with an image-crate fallback for MJPEG streams that +//! libyuv rejects). use std::io; use std::path::Path; @@ -85,10 +85,10 @@ pub(super) struct Session { device: Device, stream: MmapStream<'static>, format: DeviceFormat, - /// Driver-reported row stride in bytes (V4L2 `bytesperline`). + // Driver-reported row stride in bytes (V4L2 `bytesperline`). stride: u32, started_at: Instant, - /// Frame pulled while starting the stream, handed out first. + // Frame pulled while starting the stream, handed out first. pending_frame: Option, } diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs index 76544ff15..92ba19b18 100644 --- a/livekit-capture/src/sources/gstreamer.rs +++ b/livekit-capture/src/sources/gstreamer.rs @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Encoded video capture from a GStreamer pipeline. +//! +//! [`GStreamerVideoSource`] owns a pipeline that ends in an appsink and +//! yields the pipeline's encoded output as access units. + use ::gstreamer as gst; use ::gstreamer_app as gst_app; use bytes::Bytes; @@ -72,21 +77,22 @@ impl GStreamerSampleFormat { pub struct GStreamerVideoSourceConfig { /// GStreamer launch description for the encoded producer pipeline. /// - /// Must contain `appsink name=lk_appsink`, or leave exactly one encoded - /// video source pad unlinked for the source to attach one to. + /// The pipeline must contain `appsink name=lk_appsink`, or leave exactly + /// one encoded video source pad unlinked. The source then attaches an + /// appsink to that pad. pub pipeline: String, - /// Codec expected from the pipeline; inferred from pipeline caps when - /// omitted. + /// Codec expected from the pipeline. When omitted, the codec is + /// inferred from the pipeline caps. #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] pub codec: Option, /// Encoded frame resolution. /// - /// When omitted, the resolution is discovered from the first sample's - /// negotiated caps — construction then waits for the pipeline to produce - /// data. When set, construction returns without waiting, and the first - /// sample is verified against the declared resolution. + /// When omitted, the resolution is discovered from the first sample, so + /// construction waits for the pipeline to produce data. When set, + /// construction returns without waiting, and the first sample is + /// verified against this value. #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] pub resolution: Option, @@ -105,11 +111,12 @@ pub struct GStreamerVideoSourceConfig { )] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct GStreamerRateControlConfig { - /// Name of the encoder element in the pipeline (e.g. `lk_encoder`). + /// Name of the encoder element in the pipeline (for example + /// `lk_encoder`). pub element: String, - /// Bitrate property to set on the element (e.g. `bitrate` for x264enc, - /// `target-bitrate` for vp8enc/vp9enc). + /// Bitrate property to set on the element (for example `bitrate` for + /// x264enc, or `target-bitrate` for vp8enc/vp9enc). pub property: String, /// Unit the property expects. @@ -191,7 +198,7 @@ const DISCOVERY_TIMEOUT: gst::ClockTime = gst::ClockTime::from_seconds(5); /// Fallback frame interval when neither caps nor buffers carry timing. const DEFAULT_FRAME_INTERVAL_US: i64 = 1_000_000 / 30; -/// Encoded source that owns a GStreamer pipeline ending in an appsink. +/// Encoded source that owns a GStreamer pipeline that ends in an appsink. #[derive(Debug)] pub struct GStreamerVideoSource { pipeline: gst::Pipeline, @@ -202,36 +209,34 @@ pub struct GStreamerVideoSource { frame_interval_us: i64, next_fallback_timestamp_us: i64, rate_control: Option, - /// Caps the stream has been validated against; a pointer change on a - /// later sample triggers revalidation. + // Caps the stream was validated against; a pointer change on a later + // sample triggers revalidation. negotiated_caps: Option, - /// Sample pulled during stream discovery, handed out first. + // Sample pulled during stream discovery, handed out first. pending_sample: Option, } impl GStreamerVideoSource { - /// Creates the source, running blocking construction and stream - /// discovery on the tokio blocking pool. + /// Creates the source. Construction and stream discovery run on the + /// tokio blocking pool. /// - /// Requires a running tokio runtime. This is the async-constructor - /// convention for capture backends: `new` for async consumers, and - /// [`GStreamerVideoSource::new_blocking`] for everything else. + /// Requires a running tokio runtime. Use + /// [`GStreamerVideoSource::new_blocking`] outside of async contexts. #[cfg(feature = "tokio")] pub async fn new(config: GStreamerVideoSourceConfig) -> Result { crate::utils::run_blocking(move || Self::new_blocking(config)).await } - /// Builds, owns, and starts a GStreamer pipeline from configuration. + /// Builds and starts the GStreamer pipeline from the configuration. /// - /// The pipeline is set to `Playing` immediately — the appsink buffers a - /// bounded number of samples until a pump starts pulling — and returned - /// to `Null` when the source is dropped. Construction fails loudly on an - /// invalid launch description, a missing appsink or encoded pad, a - /// missing rate-control element, or a pipeline that refuses to start. + /// The pipeline starts to play immediately and returns to `Null` when + /// the source is dropped. Construction fails on an invalid launch + /// description, a missing appsink or encoded pad, a missing + /// rate-control element, or a pipeline that does not start. /// /// When the configuration declares no resolution, this blocks until the - /// first sample arrives (bounded by a discovery timeout) to read the - /// negotiated stream settings. + /// first sample arrives (bounded by a timeout) to read the stream + /// settings. pub fn new_blocking(config: GStreamerVideoSourceConfig) -> Result { gst::init().map_err(|err| { SourceError::new(GStreamerVideoSourceError::Pipeline(format!( @@ -316,8 +321,8 @@ impl GStreamerVideoSource { Ok(source) } - /// Blocks until the pipeline produces its first sample, surfacing bus - /// errors and bounding the wait by the discovery timeout. + /// Blocks until the pipeline produces its first sample, a bus error + /// arrives, or the discovery timeout expires. fn wait_first_sample(&self) -> Result { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(DISCOVERY_TIMEOUT.seconds()); @@ -335,12 +340,12 @@ impl GStreamerVideoSource { } } - /// Returns the owned pipeline. + /// Returns the GStreamer pipeline. pub fn pipeline(&self) -> &gst::Pipeline { &self.pipeline } - /// Surfaces a pipeline bus error, if one is pending. + /// Returns a pending pipeline bus error, if any. fn check_bus(&self) -> Result<(), GStreamerVideoSourceError> { while let Some(message) = self.bus.pop_filtered(&[gst::MessageType::Error]) { if let gst::MessageView::Error(error) = message.view() { @@ -355,15 +360,14 @@ impl GStreamerVideoSource { } /// Validates a sample's caps against the established stream settings. - /// - /// Caps are immutable and refcounted, so an unchanged stream passes with - /// a pointer comparison. On a caps change, the declared or discovered - /// resolution and codec must match: live stream reconfiguration would - /// require republishing the track, which is not supported yet. fn check_caps(&mut self, sample: &gst::Sample) -> Result<(), GStreamerVideoSourceError> { let Some(caps) = sample.caps() else { return Ok(()); }; + // Caps are immutable and refcounted, so an unchanged stream passes + // with a pointer comparison. On a caps change, the resolution and + // codec must match: live stream reconfiguration would require + // republishing the track, which is not supported yet. if let Some(seen) = &self.negotiated_caps { if seen.as_ptr() == caps.as_ptr() { return Ok(()); @@ -718,9 +722,6 @@ pub enum GStreamerPipelineError { } /// Returns the appsink caps for a codec as a launch-string fragment. -/// -/// This is the single per-codec caps table: [`encoded_caps`] and pipeline -/// descriptions embedding a capsfilter should all derive from it. pub fn encoded_caps_string(codec: EncodedVideoCodec) -> &'static str { match codec { EncodedVideoCodec::H264 => "video/x-h264,stream-format=byte-stream,alignment=au", @@ -749,7 +750,8 @@ fn sample_format_for_codec(codec: EncodedVideoCodec) -> GStreamerSampleFormat { } } -/// Returns the parser element name used to normalize a codec, when one is needed. +/// Returns the GStreamer parser element name for a codec, when one is +/// needed. pub fn parser_name(codec: EncodedVideoCodec) -> Option<&'static str> { match codec { EncodedVideoCodec::H264 => Some("h264parse"), diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index f775b69ae..9c5ef07a3 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Ready-made capture sources. Each source is gated behind its own +//! `source-*` feature. + #[cfg(feature = "source-demo")] pub mod demo; diff --git a/livekit-capture/src/utils.rs b/livekit-capture/src/utils.rs index 19cab29d5..4a6e070ac 100644 --- a/livekit-capture/src/utils.rs +++ b/livekit-capture/src/utils.rs @@ -14,9 +14,8 @@ //! Crate-internal utilities. -/// Runs blocking source construction or enumeration on the tokio blocking -/// pool, resuming panics on the caller and surfacing join failures as source -/// errors. +/// Runs a blocking task on the tokio blocking pool. Panics resume on the +/// caller, and join failures become source errors. #[cfg(feature = "tokio")] #[allow(dead_code)] pub(crate) async fn run_blocking( From 975f8d8c22d2461051f6417b41ea32efcb61cf6f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:39:06 -0700 Subject: [PATCH 50/56] Remove default features --- livekit-capture/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 9cc4895f5..3958954a8 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -23,7 +23,7 @@ yuv-sys = { workspace = true, features = ["jpeg"], optional = true } tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] -default = ["source-demo", "source-device", "source-gstreamer"] # TODO: Remove after testing +default = [] serde = ["dep:serde"] schemars = ["dep:schemars", "serde"] From 7f6ea2661a548fcbc26e1337988fc69414e5d3c3 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:49:20 -0700 Subject: [PATCH 51/56] Add pre-release disclaimer --- livekit-capture/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 06e6c2f3b..10f4c5fc9 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -1,5 +1,9 @@ # LiveKit Capture +> [!IMPORTANT] +> This crate is currently in Developer Preview mode and not ready for production use. +> There may be bugs, and APIs and configuration options are subject to change during this period. + This crate provides video capture sources and the pumps that publish them with the LiveKit [Rust SDK](../livekit/README.md). Pick a ready-made source, or implement one small trait to add your own. The application runs and From e893c9f2c245ae8bc4beec8ac8796bff3fde3cdc Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:14:11 -0700 Subject: [PATCH 52/56] Add shader source --- Cargo.lock | 2 + livekit-capture/Cargo.toml | 3 + livekit-capture/README.md | 1 + livekit-capture/shaders/prelude.wgsl | 33 ++ livekit-capture/src/sources/mod.rs | 3 + livekit-capture/src/sources/shader.rs | 749 ++++++++++++++++++++++++++ 6 files changed, 791 insertions(+) create mode 100644 livekit-capture/shaders/prelude.wgsl create mode 100644 livekit-capture/src/sources/shader.rs diff --git a/Cargo.lock b/Cargo.lock index 42fe045fe..0569e8c72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4172,11 +4172,13 @@ dependencies = [ "objc2-core-media", "objc2-core-video", "objc2-foundation 0.3.2", + "pollster", "schemars", "serde", "thiserror 2.0.19", "tokio", "v4l", + "wgpu", "yuv-sys", ] diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 3958954a8..377cf7911 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -13,10 +13,12 @@ gstreamer = { version = "0.25.2", optional = true } gstreamer-app = { version = "0.25.2", optional = true } livekit = { workspace = true } log = { workspace = true } +pollster = { version = "0.4", optional = true } schemars = { workspace = true, optional = true } serde = { workspace = true, features = ["derive"], optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["sync"] } +wgpu = { workspace = true, optional = true } yuv-sys = { workspace = true, features = ["jpeg"], optional = true } [dev-dependencies] @@ -74,6 +76,7 @@ source-device = [ "dep:libc", "dep:v4l", ] +source-shader = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 10f4c5fc9..60c32743c 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -62,3 +62,4 @@ named `source-`. Each module documents its source. | `source-demo` | `DemoVideoSource` | pixel | | `source-device` | `DeviceVideoSource` | pixel | | `source-gstreamer` | `GStreamerVideoSource` | encoded | +| `source-shader` | `ShaderVideoSource` | pixel | diff --git a/livekit-capture/shaders/prelude.wgsl b/livekit-capture/shaders/prelude.wgsl new file mode 100644 index 000000000..fb846623c --- /dev/null +++ b/livekit-capture/shaders/prelude.wgsl @@ -0,0 +1,33 @@ +// Prelude prepended to every user fragment snippet by the shader video +// source. It declares the uniforms, draws one triangle that covers the +// full target, and calls the snippet's `shade` function once per pixel. +// +// The user snippet must define `fn shade(uv: vec2) -> vec4`, +// and must not redeclare the names below. + +struct LkUniforms { + resolution: vec2, + time_s: f32, + frame_index: u32, +} + +@group(0) @binding(0) var lk: LkUniforms; + +struct LkVertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> LkVertexOutput { + let corner = vec2(f32((vertex_index << 1u) & 2u), f32(vertex_index & 2u)); + var out: LkVertexOutput; + out.position = vec4(corner * 2.0 - 1.0, 0.0, 1.0); + out.uv = vec2(corner.x, 1.0 - corner.y); + return out; +} + +@fragment +fn fs_main(in: LkVertexOutput) -> @location(0) vec4 { + return shade(in.uv); +} diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index 9c5ef07a3..aaa80776b 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -23,3 +23,6 @@ pub mod device; #[cfg(feature = "source-gstreamer")] pub mod gstreamer; + +#[cfg(feature = "source-shader")] +pub mod shader; diff --git a/livekit-capture/src/sources/shader.rs b/livekit-capture/src/sources/shader.rs new file mode 100644 index 000000000..687ba5a16 --- /dev/null +++ b/livekit-capture/src/sources/shader.rs @@ -0,0 +1,749 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Generative video from a WGSL shader. +//! +//! [`ShaderVideoSource`] renders each frame on the GPU with a WGSL shader +//! and yields the result as pixel video. The application supplies a +//! fragment snippet, inline or from a file ([`WgslShader`]). Rendering is +//! offscreen through [wgpu], so the source needs no window or display. +//! +//! The source reads each frame back from the GPU and converts it to I420 +//! on the CPU. +//! +//! [wgpu]: https://wgpu.rs + +use crate::{ + error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop, +}; +use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; +use std::{ + borrow::Cow, + fmt, + path::PathBuf, + sync::{mpsc, Arc, Mutex}, + thread, + time::{Duration, Instant}, +}; +use thiserror::Error; + +/// Render target format. Its memory layout (B, G, R, A) is the layout +/// libyuv names ARGB. +const TARGET_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Bgra8Unorm; + +/// Bytes per pixel of [`TARGET_FORMAT`]. +const TARGET_BYTES_PER_PIXEL: u32 = 4; + +/// Size of the uniform block: `vec2` + `f32` + `u32`. +const UNIFORM_BUFFER_SIZE: u64 = 16; + +/// Upper bound on one blocking GPU wait, so the stop token is observed +/// promptly. +const STOP_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Total time to wait for one frame readback before the source fails. +const READBACK_TIMEOUT: Duration = Duration::from_secs(5); + +/// Prelude prepended to a [`WgslShader::Fragment`] snippet. It draws one +/// triangle that covers the full target and calls `shade` per pixel. +const FRAGMENT_PRELUDE: &str = include_str!("../../shaders/prelude.wgsl"); + +/// WGSL fragment snippet for a [`ShaderVideoSource`]. +/// +/// The snippet must define `fn shade(uv: vec2) -> vec4`. The +/// source calls `shade` once per pixel. `uv` runs from `(0.0, 0.0)` at +/// the top left to `(1.0, 1.0)` at the bottom right. The return value is +/// the pixel color as RGBA in the 0.0 to 1.0 range. Alpha is ignored. +/// +/// A fixed prelude declares these uniforms for the snippet: +/// +/// ```wgsl +/// struct LkUniforms { +/// resolution: vec2, // output size in pixels +/// time_s: f32, // seconds since the stream started +/// frame_index: u32, // frame counter, starts at 0 +/// } +/// @group(0) @binding(0) var lk: LkUniforms; +/// ``` +/// +/// The prelude reserves the names `lk`, `LkUniforms`, `LkVertexOutput`, +/// `vs_main`, and `fs_main`. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "snake_case") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum WgslShader { + /// Inline fragment snippet. + Fragment(String), + /// Path to a file that contains a fragment snippet. + /// + /// The source reads the file once, at construction. A relative path + /// resolves against the process working directory. + FragmentFile(PathBuf), +} + +impl WgslShader { + /// Returns the complete WGSL module to compile, reading the snippet + /// from disk when needed. + fn module_code(&self) -> Result { + let snippet = match self { + Self::Fragment(snippet) => Cow::Borrowed(snippet.as_str()), + Self::FragmentFile(path) => { + Cow::Owned(std::fs::read_to_string(path).map_err(|error| { + ShaderVideoSourceError::ShaderFile { path: path.clone(), error } + })?) + } + }; + Ok(format!("{FRAGMENT_PRELUDE}\n{snippet}")) + } +} + +/// Configuration for a [`ShaderVideoSource`]. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct ShaderVideoSourceConfig { + /// Output resolution. + pub resolution: VideoResolution, + /// Output frame rate in frames per second. + pub framerate_fps: u32, + /// Shader that colors each frame. + pub shader: WgslShader, +} + +/// Error returned by a shader video source. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ShaderVideoSourceError { + /// The configured resolution has a zero component. + #[error("shader source resolution must be non-zero")] + ZeroResolution, + /// The configured frame rate is zero. + #[error("shader source frame rate must be non-zero")] + ZeroFramerate, + /// No compatible GPU adapter is available. + #[error("no compatible GPU adapter: {0}")] + NoAdapter(String), + /// The GPU adapter rejected the device request. + #[error("failed to open the GPU device: {0}")] + Device(String), + /// The shader file could not be read. + #[error("failed to read shader file '{}': {error}", path.display())] + ShaderFile { + /// Path of the shader file. + path: PathBuf, + /// The underlying read error. + #[source] + error: std::io::Error, + }, + /// The shader or its pipeline failed to build. + #[error("failed to build the shader pipeline: {0}")] + ShaderCompile(String), + /// The GPU reported an error. + #[error("GPU error: {0}")] + Backend(String), + /// Reading the rendered frame back from the GPU failed. + #[error("failed to read the frame back from the GPU: {0}")] + Readback(String), + /// Pixel conversion failed. + #[error("failed to convert the rendered frame to I420: {0}")] + Convert(&'static str), +} + +/// Pixel video source that renders each frame on the GPU with a WGSL +/// shader. +/// +/// Construction selects a GPU adapter and compiles the shader, so a bad +/// shader fails construction, not the pump. The source sleeps to pace +/// itself to the configured frame rate. It never reaches the end of its +/// stream — stop the pump that drives it instead. +pub struct ShaderVideoSource { + config: ShaderVideoSourceConfig, + renderer: ShaderRenderer, + started: Option, + frame_index: u64, +} + +impl ShaderVideoSource { + /// Creates the source. GPU setup runs on the tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`ShaderVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: ShaderVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await + } + + /// Selects a GPU adapter, compiles the shader, and builds the render + /// pipeline. + /// + /// Construction fails when no GPU is available, when the shader does + /// not compile, when the shader file cannot be read, or for a zero + /// resolution or frame rate. + pub fn new_blocking(config: ShaderVideoSourceConfig) -> Result { + validate_config(&config).map_err(SourceError::new)?; + let renderer = ShaderRenderer::new(&config).map_err(SourceError::new)?; + Ok(Self { config, renderer, started: None, frame_index: 0 }) + } + + /// Returns the configuration the source was created with. + pub fn config(&self) -> &ShaderVideoSourceConfig { + &self.config + } + + fn frame_interval(&self) -> Duration { + Duration::from_secs(1) / self.config.framerate_fps + } +} + +impl PixelVideoSource for ShaderVideoSource { + fn resolution(&self) -> VideoResolution { + self.config.resolution + } + + // The pacing sleep is at most one frame interval, and every readback + // wait is bounded by STOP_POLL_INTERVAL, so the stop token is + // observed promptly. + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + let started = *self.started.get_or_insert_with(Instant::now); + + // Pace against the ideal timeline so timestamps stay jitter-free. + let interval_us = self.frame_interval().as_micros() as u64; + let elapsed = Duration::from_micros(self.frame_index.saturating_mul(interval_us)); + let due = started + elapsed; + if let Some(wait) = due.checked_duration_since(Instant::now()) { + thread::sleep(wait); + } + + // The uniform frame index wraps after u32::MAX frames. + let frame_index = self.frame_index as u32; + self.frame_index += 1; + let buffer = self + .renderer + .render_frame(elapsed.as_secs_f32(), frame_index, stop) + .map_err(SourceError::new)?; + let Some(buffer) = buffer else { + // The stop token fired during the readback wait. + return Ok(None); + }; + + Ok(Some(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: elapsed.as_micros() as i64, + frame_metadata: None, + buffer: Box::new(buffer), + })) + } +} + +impl fmt::Debug for ShaderVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ShaderVideoSource") + .field("config", &self.config) + .field("frame_index", &self.frame_index) + .finish_non_exhaustive() + } +} + +/// Validates the CPU-checkable parts of a configuration. +fn validate_config(config: &ShaderVideoSourceConfig) -> Result<(), ShaderVideoSourceError> { + let VideoResolution { width, height } = config.resolution; + if width == 0 || height == 0 { + return Err(ShaderVideoSourceError::ZeroResolution); + } + if config.framerate_fps == 0 { + return Err(ShaderVideoSourceError::ZeroFramerate); + } + Ok(()) +} + +/// Owns the wgpu state and renders one frame at a time. +struct ShaderRenderer { + device: wgpu::Device, + queue: wgpu::Queue, + pipeline: wgpu::RenderPipeline, + bind_group: wgpu::BindGroup, + uniform_buffer: wgpu::Buffer, + target: wgpu::Texture, + target_view: wgpu::TextureView, + /// Readback destination, reused across frames. Rows are padded to + /// the wgpu copy alignment. + staging: wgpu::Buffer, + padded_bytes_per_row: u32, + resolution: VideoResolution, + /// First uncaptured GPU error, stashed by the device error handler + /// and surfaced on the next frame. + device_error: Arc>>, +} + +impl ShaderRenderer { + fn new(config: &ShaderVideoSourceConfig) -> Result { + let VideoResolution { width, height } = config.resolution; + // Read and assemble the shader first, so a missing file fails + // before any GPU setup. + let module_code = config.shader.module_code()?; + let padded_bytes_per_row = padded_bytes_per_row(width) + .ok_or_else(|| ShaderVideoSourceError::Backend("resolution is too large".to_owned()))?; + + // Rendering is offscreen, so no display handle is needed. WGPU_* + // environment variables can override the backend and adapter + // selection. + let instance = + wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::from_env().unwrap_or_default(), + ..Default::default() + })) + .map_err(|err| ShaderVideoSourceError::NoAdapter(err.to_string()))?; + + let info = adapter.get_info(); + log::info!("Rendering shader source on \"{}\" ({})", info.name, info.backend); + + // Clamp the default limits to what the adapter supports, so weaker + // adapters (GL, software rasterizers) still open. A resolution + // beyond the clamped limits fails texture creation below. + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { + label: Some("lk_shader_device"), + required_limits: wgpu::Limits::default().or_worse_values_from(&adapter.limits()), + ..Default::default() + })) + .map_err(|err| ShaderVideoSourceError::Device(err.to_string()))?; + + // Runtime GPU errors have no return channel of their own: stash + // the first one and report it from the next render_frame call. + let device_error: Arc>> = Arc::default(); + let sink = Arc::clone(&device_error); + device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { + log::error!("shader source GPU error: {error}"); + let mut slot = sink.lock().unwrap(); + if slot.is_none() { + *slot = Some(error.to_string()); + } + })); + + // Compile the shader and build the pipeline under an error scope, + // so a bad shader fails construction with its compile message. + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("lk_shader_module"), + source: wgpu::ShaderSource::Wgsl(module_code.into()), + }); + let bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("lk_shader_bind_group_layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu::BufferSize::new(UNIFORM_BUFFER_SIZE), + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("lk_shader_pipeline_layout"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("lk_shader_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: TARGET_FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + if let Some(error) = pollster::block_on(scope.pop()) { + return Err(ShaderVideoSourceError::ShaderCompile(error.to_string())); + } + + // Build the target and readback resources under their own scope, + // so an unsupported resolution also fails construction. + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let target = device.create_texture(&wgpu::TextureDescriptor { + label: Some("lk_shader_target"), + size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: TARGET_FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let target_view = target.create_view(&wgpu::TextureViewDescriptor::default()); + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("lk_shader_uniforms"), + size: UNIFORM_BUFFER_SIZE, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("lk_shader_bind_group"), + layout: &bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }], + }); + let staging = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("lk_shader_staging"), + size: u64::from(padded_bytes_per_row) * u64::from(height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + if let Some(error) = pollster::block_on(scope.pop()) { + return Err(ShaderVideoSourceError::Backend(error.to_string())); + } + + Ok(Self { + device, + queue, + pipeline, + bind_group, + uniform_buffer, + target, + target_view, + staging, + padded_bytes_per_row, + resolution: config.resolution, + device_error, + }) + } + + /// Renders one frame and reads it back as I420. Returns `Ok(None)` + /// when the stop token fires during the readback wait. + fn render_frame( + &self, + time_s: f32, + frame_index: u32, + stop: &PumpStop, + ) -> Result, ShaderVideoSourceError> { + self.check_device_error()?; + + let VideoResolution { width, height } = self.resolution; + self.queue.write_buffer( + &self.uniform_buffer, + 0, + &uniform_bytes(self.resolution, time_s, frame_index), + ); + + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("lk_shader") }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("lk_shader_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.target_view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + ..Default::default() + }); + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &self.bind_group, &[]); + pass.draw(0..3, 0..1); + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.target, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &self.staging, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.padded_bytes_per_row), + rows_per_image: None, + }, + }, + wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + ); + // Schedule the mapping with the submission, so no separate + // map_async call is needed after submit. + let (mapped_tx, mapped_rx) = mpsc::channel(); + encoder.map_buffer_on_submit(&self.staging, wgpu::MapMode::Read, .., move |result| { + let _ = mapped_tx.send(result); + }); + let submission = self.queue.submit([encoder.finish()]); + + if !self.wait_for_map(submission, &mapped_rx, stop)? { + // Stopped: cancel the pending mapping to leave the buffer + // reusable. + self.staging.unmap(); + return Ok(None); + } + + let mapped = self.staging.slice(..).get_mapped_range(); + let converted = convert_to_i420(&mapped, self.padded_bytes_per_row, width, height); + drop(mapped); + self.staging.unmap(); + converted.map(Some) + } + + /// Waits for the staging buffer to be mapped. Returns `Ok(false)` when + /// the stop token fires first. + fn wait_for_map( + &self, + submission: wgpu::SubmissionIndex, + mapped: &mpsc::Receiver>, + stop: &PumpStop, + ) -> Result { + let deadline = Instant::now() + READBACK_TIMEOUT; + loop { + let poll = self.device.poll(wgpu::PollType::Wait { + submission_index: Some(submission.clone()), + timeout: Some(STOP_POLL_INTERVAL), + }); + match poll { + Ok(_) | Err(wgpu::PollError::Timeout) => {} + Err(err) => return Err(ShaderVideoSourceError::Readback(err.to_string())), + } + match mapped.try_recv() { + Ok(Ok(())) => return Ok(true), + Ok(Err(err)) => return Err(ShaderVideoSourceError::Readback(err.to_string())), + Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => { + return Err(ShaderVideoSourceError::Readback( + "map callback was dropped".to_owned(), + )); + } + } + self.check_device_error()?; + if stop.is_stopped() { + return Ok(false); + } + if Instant::now() >= deadline { + return Err(ShaderVideoSourceError::Readback( + "timed out waiting for the GPU".to_owned(), + )); + } + } + } + + /// Reports the first stashed GPU error, if there is one. + fn check_device_error(&self) -> Result<(), ShaderVideoSourceError> { + match &*self.device_error.lock().unwrap() { + Some(message) => Err(ShaderVideoSourceError::Backend(message.clone())), + None => Ok(()), + } + } +} + +/// Serializes the uniform block: resolution, time, and frame index. +fn uniform_bytes( + resolution: VideoResolution, + time_s: f32, + frame_index: u32, +) -> [u8; UNIFORM_BUFFER_SIZE as usize] { + let mut bytes = [0u8; UNIFORM_BUFFER_SIZE as usize]; + bytes[0..4].copy_from_slice(&(resolution.width as f32).to_ne_bytes()); + bytes[4..8].copy_from_slice(&(resolution.height as f32).to_ne_bytes()); + bytes[8..12].copy_from_slice(&time_s.to_ne_bytes()); + bytes[12..16].copy_from_slice(&frame_index.to_ne_bytes()); + bytes +} + +/// Returns the staging-buffer row stride: the pixel row size rounded up +/// to the wgpu copy alignment. `None` when the value overflows `u32`. +fn padded_bytes_per_row(width: u32) -> Option { + let unpadded = u64::from(width) * u64::from(TARGET_BYTES_PER_PIXEL); + let align = u64::from(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT); + u32::try_from(unpadded.div_ceil(align) * align).ok() +} + +/// Converts one padded BGRA image to a freshly allocated I420 buffer. +fn convert_to_i420( + source: &[u8], + source_stride: u32, + width: u32, + height: u32, +) -> Result { + if source.len() < source_stride as usize * height as usize { + return Err(ShaderVideoSourceError::Convert("mapped frame is too short")); + } + let source_stride = i32::try_from(source_stride) + .map_err(|_| ShaderVideoSourceError::Convert("stride exceeds supported range"))?; + let width_i32 = i32::try_from(width) + .map_err(|_| ShaderVideoSourceError::Convert("width exceeds supported range"))?; + let height_i32 = i32::try_from(height) + .map_err(|_| ShaderVideoSourceError::Convert("height exceeds supported range"))?; + + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers `height` rows of `source_stride` bytes, and the + // destination planes come from a freshly allocated I420Buffer with matching width, + // height, and strides. + let ret = unsafe { + yuv_sys::rs_ARGBToI420( + source.as_ptr(), + source_stride, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + }; + if ret != 0 { + return Err(ShaderVideoSourceError::Convert("ARGBToI420 failed")); + } + Ok(buffer) +} + +#[cfg(test)] +mod tests { + use super::*; + + const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; + + fn red_config() -> ShaderVideoSourceConfig { + ShaderVideoSourceConfig { + resolution: RESOLUTION, + framerate_fps: 200, + shader: WgslShader::Fragment( + "fn shade(uv: vec2) -> vec4 { return vec4(1.0, 0.0, 0.0, 1.0); }" + .to_owned(), + ), + } + } + + /// GPU tests skip when the machine has no usable adapter. + fn gpu_available() -> bool { + let instance = + wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); + pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) + .is_ok() + } + + #[test] + fn validation_rejects_zero_resolution_and_framerate() { + let mut config = red_config(); + config.resolution = VideoResolution::new(0, 36); + assert!(matches!( + validate_config(&config), + Err(ShaderVideoSourceError::ZeroResolution) + )); + + let mut config = red_config(); + config.framerate_fps = 0; + assert!(matches!(validate_config(&config), Err(ShaderVideoSourceError::ZeroFramerate))); + } + + #[test] + fn fragment_snippets_are_appended_to_the_prelude() { + let shader = WgslShader::Fragment("fn shade(...) {}".to_owned()); + let code = shader.module_code().unwrap(); + assert!(code.contains("fn vs_main")); + assert!(code.contains("fn fs_main")); + assert!(code.ends_with("fn shade(...) {}")); + } + + #[test] + fn fragment_files_are_read_from_disk() { + let path = std::env::temp_dir().join(format!("lk_shader_test_{}.wgsl", std::process::id())); + std::fs::write(&path, "fn shade(...) {}").unwrap(); + let code = WgslShader::FragmentFile(path.clone()).module_code(); + std::fs::remove_file(&path).ok(); + assert!(code.unwrap().ends_with("fn shade(...) {}")); + } + + #[test] + fn missing_fragment_files_fail() { + let shader = WgslShader::FragmentFile(PathBuf::from("lk_shader_test_missing.wgsl")); + assert!(matches!( + shader.module_code(), + Err(ShaderVideoSourceError::ShaderFile { .. }) + )); + } + + #[test] + fn rows_are_padded_to_the_copy_alignment() { + assert_eq!(padded_bytes_per_row(64), Some(256)); + assert_eq!(padded_bytes_per_row(321), Some(1536)); + assert_eq!(padded_bytes_per_row(1280), Some(5120)); + assert_eq!(padded_bytes_per_row(u32::MAX), None); + } + + #[test] + fn renders_solid_color_frames_at_the_frame_rate() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut source = ShaderVideoSource::new_blocking(red_config()).unwrap(); + + let stop = PumpStop::new(); + let first = source.next_frame(&stop).unwrap().unwrap(); + let second = source.next_frame(&stop).unwrap().unwrap(); + assert_eq!((first.buffer.width(), first.buffer.height()), (64, 36)); + assert_eq!(first.timestamp_us, 0); + assert_eq!(second.timestamp_us, 5_000); + + // Red in limited-range BT.601, as converted by libyuv. + let i420 = first.buffer.as_i420().expect("shader source yields I420 buffers"); + let (y, u, v) = i420.data(); + assert!(y[0].abs_diff(82) <= 2, "unexpected luma {}", y[0]); + assert!(u[0].abs_diff(90) <= 2, "unexpected chroma-u {}", u[0]); + assert!(v[0].abs_diff(240) <= 2, "unexpected chroma-v {}", v[0]); + } + + #[test] + fn invalid_shaders_fail_construction() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut config = red_config(); + config.shader = WgslShader::Fragment("this is not wgsl".to_owned()); + assert!(ShaderVideoSource::new_blocking(config).is_err()); + } +} From e5cf95d6d7fb45f5840bca8276c86cca440da45d Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:24:29 -0700 Subject: [PATCH 53/56] Make demo source shader backed --- livekit-capture/Cargo.toml | 2 +- livekit-capture/README.md | 2 +- livekit-capture/shaders/demo.wgsl | 5 + livekit-capture/src/sources/demo.rs | 192 ++++++++------------------ livekit-capture/src/sources/shader.rs | 17 +-- livekit-ffi/protocol/capture.proto | 2 +- livekit-ffi/src/server/capture.rs | 1 + 7 files changed, 74 insertions(+), 147 deletions(-) create mode 100644 livekit-capture/shaders/demo.wgsl diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 377cf7911..b068f0305 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -34,7 +34,7 @@ schemars = ["dep:schemars", "serde"] tokio = ["tokio/rt"] # Pixel sources -source-demo = [] +source-demo = ["source-shader"] source-device = [ "dep:yuv-sys", # macOS backend diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 60c32743c..56410e7e9 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -40,7 +40,7 @@ A pump supplies the RTC source and the publish options, so publication is the same for either path. ```rust -let pump = PixelVideoPump::new(DemoVideoSource::new(config)?); +let pump = PixelVideoPump::new(DemoVideoSource::new(config).await?); let track = LocalVideoTrack::create_video_track("demo", pump.rtc_source()); let options = pump.publish_options(); diff --git a/livekit-capture/shaders/demo.wgsl b/livekit-capture/shaders/demo.wgsl new file mode 100644 index 000000000..c65efd64c --- /dev/null +++ b/livekit-capture/shaders/demo.wgsl @@ -0,0 +1,5 @@ +// Animated color gradient rendered by the demo video source. +fn shade(uv: vec2) -> vec4 { + let color = 0.5 + 0.5 * cos(lk.time_s + uv.xyx * 4.0 + vec3(0.0, 2.0, 4.0)); + return vec4(color, 1.0); +} diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs index 4dcec8a23..cc0ff6302 100644 --- a/livekit-capture/src/sources/demo.rs +++ b/livekit-capture/src/sources/demo.rs @@ -12,30 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Solid-color demo video source for testing. +//! Demo video source for testing: an animated color gradient. use crate::{ - error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop, + error::SourceError, + pixel::PixelVideoSource, + primitive::VideoResolution, + pump::PumpStop, + sources::shader::{ShaderVideoSource, ShaderVideoSourceConfig, WgslShader}, }; -use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; -use std::{ - thread, - time::{Duration, Instant}, -}; -use thiserror::Error; - -/// How long each palette color is shown before cycling to the next. -const COLOR_INTERVAL: Duration = Duration::from_millis(500); +use livekit::webrtc::video_frame::BoxVideoFrame; -/// Colors the demo source cycles through, as `(r, g, b)`. -const PALETTE: [(u8, u8, u8); 6] = [ - (0xE6, 0x32, 0x2E), // red - (0xF4, 0x9D, 0x1A), // orange - (0xF7, 0xD0, 0x38), // yellow - (0x2E, 0xB8, 0x5C), // green - (0x2E, 0x6F, 0xE6), // blue - (0x8E, 0x44, 0xAD), // purple -]; +/// Fragment snippet the demo source renders. +const DEMO_SHADER: &str = include_str!("../../shaders/demo.wgsl"); /// Configuration for a [`DemoVideoSource`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -52,103 +41,61 @@ pub struct DemoVideoSourceConfig { pub framerate_fps: u32, } -/// Pixel video source that produces solid-color frames from a fixed -/// palette. +/// Pixel video source that renders an animated color gradient. /// -/// The source sleeps to pace itself to the configured frame rate. It never -/// reaches the end of its stream — stop the pump that drives it instead. +/// This is a convenience wrapper around [`ShaderVideoSource`] with a +/// built-in shader. The source paces itself to the configured frame +/// rate. It never reaches the end of its stream — stop the pump that +/// drives it instead. #[derive(Debug)] pub struct DemoVideoSource { config: DemoVideoSourceConfig, - // One `(y, u, v)` sample triple per palette color. - colors: [(u8, u8, u8); PALETTE.len()], - started: Option, - frame_index: u64, + inner: ShaderVideoSource, } impl DemoVideoSource { - /// Creates a demo source. Returns an error for a zero resolution or - /// frame rate. - pub fn new(config: DemoVideoSourceConfig) -> Result { - let VideoResolution { width, height } = config.resolution; - if width == 0 || height == 0 { - return Err(SourceError::new(DemoVideoSourceConfigError::ZeroResolution)); - } - if config.framerate_fps == 0 { - return Err(SourceError::new(DemoVideoSourceConfigError::ZeroFramerate)); - } - - let colors = PALETTE.map(yuv_from_rgb); - Ok(Self { config, colors, started: None, frame_index: 0 }) + /// Creates the source. GPU setup runs on the tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`DemoVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: DemoVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await } - fn frame_interval(&self) -> Duration { - Duration::from_secs(1) / self.config.framerate_fps + /// Creates the source and its GPU state. + /// + /// Construction fails when no GPU is available, or for a zero + /// resolution or frame rate. + pub fn new_blocking(config: DemoVideoSourceConfig) -> Result { + let inner = ShaderVideoSource::new_blocking(ShaderVideoSourceConfig { + resolution: config.resolution, + framerate_fps: config.framerate_fps, + shader: WgslShader::Fragment(DEMO_SHADER.to_owned()), + })?; + Ok(Self { config, inner }) } -} -/// Error returned for an invalid [`DemoVideoSourceConfig`]. -#[derive(Debug, Error)] -pub enum DemoVideoSourceConfigError { - /// The configured resolution has a zero component. - #[error("demo source resolution must be non-zero")] - ZeroResolution, - /// The configured frame rate is zero. - #[error("demo source frame rate must be non-zero")] - ZeroFramerate, + /// Returns the configuration the source was created with. + pub fn config(&self) -> DemoVideoSourceConfig { + self.config + } } impl PixelVideoSource for DemoVideoSource { fn resolution(&self) -> VideoResolution { - self.config.resolution + self.inner.resolution() } - // Sleeps at most one frame interval, so the stop token is observed - // promptly without integrating it into the wait. - fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { - let started = *self.started.get_or_insert_with(Instant::now); - - // Pace against the ideal timeline so timestamps stay jitter-free. - let interval_us = self.frame_interval().as_micros() as u64; - let elapsed = Duration::from_micros(self.frame_index.saturating_mul(interval_us)); - let due = started + elapsed; - if let Some(wait) = due.checked_duration_since(Instant::now()) { - thread::sleep(wait); - } - - let timestamp_us = elapsed.as_micros() as i64; - let color_index = (elapsed.as_micros() / COLOR_INTERVAL.as_micros()) as usize; - let (y, u, v) = self.colors[color_index % self.colors.len()]; - - self.frame_index += 1; - let VideoResolution { width, height } = self.config.resolution; - let mut buffer = I420Buffer::new(width, height); - let (data_y, data_u, data_v) = buffer.data_mut(); - data_y.fill(y); - data_u.fill(u); - data_v.fill(v); - - Ok(Some(VideoFrame { - rotation: VideoRotation::VideoRotation0, - timestamp_us, - frame_metadata: None, - buffer: Box::new(buffer), - })) + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + self.inner.next_frame(stop) } } -/// Converts an RGB color to limited-range BT.601 YUV. -fn yuv_from_rgb((r, g, b): (u8, u8, u8)) -> (u8, u8, u8) { - let (r, g, b) = (r as f32 / 255.0, g as f32 / 255.0, b as f32 / 255.0); - let y = 16.0 + 65.481 * r + 128.553 * g + 24.966 * b; - let u = 128.0 - 37.797 * r - 74.203 * g + 112.0 * b; - let v = 128.0 + 112.0 * r - 93.786 * g - 18.214 * b; - (y.round() as u8, u.round() as u8, v.round() as u8) -} - #[cfg(test)] mod tests { use super::*; + use crate::sources::shader::gpu_available; fn test_config() -> DemoVideoSourceConfig { DemoVideoSourceConfig { @@ -158,55 +105,28 @@ mod tests { } #[test] - fn yields_frames_with_configured_dimensions() { - let mut source = DemoVideoSource::new(test_config()).unwrap(); - - let frame = source.next_frame(&PumpStop::new()).unwrap().unwrap(); - assert_eq!((frame.buffer.width(), frame.buffer.height()), (64, 36)); - - let i420 = frame.buffer.as_i420().expect("demo source yields I420 buffers"); - let (y, u, v) = i420.data(); - assert_eq!(y.len(), 64 * 36); - assert_eq!(u.len(), 32 * 18); - assert_eq!(v.len(), 32 * 18); + fn rejects_zero_configuration() { + let mut config = test_config(); + config.resolution = VideoResolution::new(0, 36); + assert!(DemoVideoSource::new_blocking(config).is_err()); + + let mut config = test_config(); + config.framerate_fps = 0; + assert!(DemoVideoSource::new_blocking(config).is_err()); } #[test] - fn timestamps_follow_the_frame_rate() { - let mut source = DemoVideoSource::new(test_config()).unwrap(); + fn yields_frames_with_configured_dimensions() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut source = DemoVideoSource::new_blocking(test_config()).unwrap(); let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); let second = source.next_frame(&PumpStop::new()).unwrap().unwrap(); + assert_eq!((first.buffer.width(), first.buffer.height()), (64, 36)); assert_eq!(first.timestamp_us, 0); assert_eq!(second.timestamp_us, 1_000); } - - #[test] - fn colors_cycle_at_the_color_interval() { - // Two frames per color, so the first boundary lands on frame three. - // The source paces itself in real time, so this trades frame count - // for the ~COLOR_INTERVAL the test spends sleeping either way. - let frame_interval = COLOR_INTERVAL / 2; - let mut source = DemoVideoSource::new(DemoVideoSourceConfig { - resolution: VideoResolution { width: 64, height: 36 }, - framerate_fps: (Duration::from_secs(1).as_micros() / frame_interval.as_micros()) as u32, - }) - .unwrap(); - - let luma = |frame: &BoxVideoFrame| frame.buffer.as_i420().unwrap().data().0[0]; - - let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); - let same_color = source.next_frame(&PumpStop::new()).unwrap().unwrap(); - let next_color = source.next_frame(&PumpStop::new()).unwrap().unwrap(); - assert_eq!(luma(&first), luma(&same_color)); - assert_ne!(luma(&first), luma(&next_color)); - } - - #[test] - fn converts_primaries_to_expected_luma() { - // White has maximum luma and centered chroma in limited range. - assert_eq!(yuv_from_rgb((255, 255, 255)), (235, 128, 128)); - // Black has minimum luma and centered chroma. - assert_eq!(yuv_from_rgb((0, 0, 0)), (16, 128, 128)); - } } diff --git a/livekit-capture/src/sources/shader.rs b/livekit-capture/src/sources/shader.rs index 687ba5a16..3c6763d83 100644 --- a/livekit-capture/src/sources/shader.rs +++ b/livekit-capture/src/sources/shader.rs @@ -596,6 +596,15 @@ fn padded_bytes_per_row(width: u32) -> Option { u32::try_from(unpadded.div_ceil(align) * align).ok() } +/// Returns whether a GPU adapter is available. Tests that need a GPU +/// skip when there is none. +#[cfg(test)] +pub(crate) fn gpu_available() -> bool { + let instance = + wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); + pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).is_ok() +} + /// Converts one padded BGRA image to a freshly allocated I420 buffer. fn convert_to_i420( source: &[u8], @@ -656,14 +665,6 @@ mod tests { } } - /// GPU tests skip when the machine has no usable adapter. - fn gpu_available() -> bool { - let instance = - wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); - pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) - .is_ok() - } - #[test] fn validation_rejects_zero_resolution_and_framerate() { let mut config = red_config(); diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index be84f6481..5f34835c9 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -72,7 +72,7 @@ message GstreamerVideoSourceConfig { optional GstreamerRateControl rate_control = 4; } -// Test source producing solid-color frames, cycling through a palette. +// Test source rendering an animated color gradient on the GPU. message DemoVideoSourceConfig { // Output resolution. required VideoSourceResolution resolution = 1; diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index af00b667b..13c6dd768 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -121,6 +121,7 @@ async fn create_capture_source( } proto::new_capture_source_request::Config::Demo(config) => { let source = DemoVideoSource::new(config.into()) + .await .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; let source: Box = Box::new(source); CapturePump::Pixel(PixelVideoPump::new(source)) From 4290afcbde5df547dc703f4044f239bf1d37a489 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:00:32 -0700 Subject: [PATCH 54/56] Pattern source Supersedes shader and demo source --- livekit-capture/Cargo.toml | 3 +- livekit-capture/README.md | 7 +- .../shaders/{demo.wgsl => gradient.wgsl} | 2 +- livekit-capture/shaders/prelude.wgsl | 9 +- livekit-capture/src/sources/demo.rs | 132 -------- livekit-capture/src/sources/mod.rs | 7 +- .../src/sources/{shader.rs => pattern.rs} | 296 +++++++----------- livekit-ffi/Cargo.toml | 2 +- livekit-ffi/protocol/capture.proto | 14 +- livekit-ffi/src/conversion/capture.rs | 18 +- livekit-ffi/src/server/capture.rs | 17 +- 11 files changed, 161 insertions(+), 346 deletions(-) rename livekit-capture/shaders/{demo.wgsl => gradient.wgsl} (72%) delete mode 100644 livekit-capture/src/sources/demo.rs rename livekit-capture/src/sources/{shader.rs => pattern.rs} (69%) diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index b068f0305..8ad337ca1 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -34,7 +34,6 @@ schemars = ["dep:schemars", "serde"] tokio = ["tokio/rt"] # Pixel sources -source-demo = ["source-shader"] source-device = [ "dep:yuv-sys", # macOS backend @@ -76,7 +75,7 @@ source-device = [ "dep:libc", "dep:v4l", ] -source-shader = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] +source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 56410e7e9..26d065525 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -40,9 +40,9 @@ A pump supplies the RTC source and the publish options, so publication is the same for either path. ```rust -let pump = PixelVideoPump::new(DemoVideoSource::new(config).await?); +let pump = PixelVideoPump::new(PatternVideoSource::new(config).await?); -let track = LocalVideoTrack::create_video_track("demo", pump.rtc_source()); +let track = LocalVideoTrack::create_video_track("pattern", pump.rtc_source()); let options = pump.publish_options(); room.local_participant().publish_track(LocalTrack::Video(track), options).await?; @@ -59,7 +59,6 @@ named `source-`. Each module documents its source. | Feature | Source | Kind | | ------------------ | ---------------------- | ------- | -| `source-demo` | `DemoVideoSource` | pixel | | `source-device` | `DeviceVideoSource` | pixel | | `source-gstreamer` | `GStreamerVideoSource` | encoded | -| `source-shader` | `ShaderVideoSource` | pixel | +| `source-pattern` | `PatternVideoSource` | pixel | diff --git a/livekit-capture/shaders/demo.wgsl b/livekit-capture/shaders/gradient.wgsl similarity index 72% rename from livekit-capture/shaders/demo.wgsl rename to livekit-capture/shaders/gradient.wgsl index c65efd64c..7e5f6c07a 100644 --- a/livekit-capture/shaders/demo.wgsl +++ b/livekit-capture/shaders/gradient.wgsl @@ -1,4 +1,4 @@ -// Animated color gradient rendered by the demo video source. +// Animated color gradient: the built-in gradient pattern. fn shade(uv: vec2) -> vec4 { let color = 0.5 + 0.5 * cos(lk.time_s + uv.xyx * 4.0 + vec3(0.0, 2.0, 4.0)); return vec4(color, 1.0); diff --git a/livekit-capture/shaders/prelude.wgsl b/livekit-capture/shaders/prelude.wgsl index fb846623c..2f74da385 100644 --- a/livekit-capture/shaders/prelude.wgsl +++ b/livekit-capture/shaders/prelude.wgsl @@ -1,8 +1,9 @@ -// Prelude prepended to every user fragment snippet by the shader video -// source. It declares the uniforms, draws one triangle that covers the -// full target, and calls the snippet's `shade` function once per pixel. +// Prelude prepended to every pattern fragment snippet by the pattern +// video source. It declares the uniforms, draws one triangle that covers +// the full target, and calls the snippet's `shade` function once per +// pixel. // -// The user snippet must define `fn shade(uv: vec2) -> vec4`, +// Each pattern snippet must define `fn shade(uv: vec2) -> vec4`, // and must not redeclare the names below. struct LkUniforms { diff --git a/livekit-capture/src/sources/demo.rs b/livekit-capture/src/sources/demo.rs deleted file mode 100644 index cc0ff6302..000000000 --- a/livekit-capture/src/sources/demo.rs +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Demo video source for testing: an animated color gradient. - -use crate::{ - error::SourceError, - pixel::PixelVideoSource, - primitive::VideoResolution, - pump::PumpStop, - sources::shader::{ShaderVideoSource, ShaderVideoSourceConfig, WgslShader}, -}; -use livekit::webrtc::video_frame::BoxVideoFrame; - -/// Fragment snippet the demo source renders. -const DEMO_SHADER: &str = include_str!("../../shaders/demo.wgsl"); - -/// Configuration for a [`DemoVideoSource`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr( - feature = "serde", - derive(serde::Serialize, serde::Deserialize), - serde(deny_unknown_fields) -)] -#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -pub struct DemoVideoSourceConfig { - /// Output resolution. - pub resolution: VideoResolution, - /// Output frame rate in frames per second. - pub framerate_fps: u32, -} - -/// Pixel video source that renders an animated color gradient. -/// -/// This is a convenience wrapper around [`ShaderVideoSource`] with a -/// built-in shader. The source paces itself to the configured frame -/// rate. It never reaches the end of its stream — stop the pump that -/// drives it instead. -#[derive(Debug)] -pub struct DemoVideoSource { - config: DemoVideoSourceConfig, - inner: ShaderVideoSource, -} - -impl DemoVideoSource { - /// Creates the source. GPU setup runs on the tokio blocking pool. - /// - /// Requires a running tokio runtime. Use - /// [`DemoVideoSource::new_blocking`] outside of async contexts. - #[cfg(feature = "tokio")] - pub async fn new(config: DemoVideoSourceConfig) -> Result { - crate::utils::run_blocking(move || Self::new_blocking(config)).await - } - - /// Creates the source and its GPU state. - /// - /// Construction fails when no GPU is available, or for a zero - /// resolution or frame rate. - pub fn new_blocking(config: DemoVideoSourceConfig) -> Result { - let inner = ShaderVideoSource::new_blocking(ShaderVideoSourceConfig { - resolution: config.resolution, - framerate_fps: config.framerate_fps, - shader: WgslShader::Fragment(DEMO_SHADER.to_owned()), - })?; - Ok(Self { config, inner }) - } - - /// Returns the configuration the source was created with. - pub fn config(&self) -> DemoVideoSourceConfig { - self.config - } -} - -impl PixelVideoSource for DemoVideoSource { - fn resolution(&self) -> VideoResolution { - self.inner.resolution() - } - - fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { - self.inner.next_frame(stop) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::sources::shader::gpu_available; - - fn test_config() -> DemoVideoSourceConfig { - DemoVideoSourceConfig { - resolution: VideoResolution { width: 64, height: 36 }, - framerate_fps: 1000, - } - } - - #[test] - fn rejects_zero_configuration() { - let mut config = test_config(); - config.resolution = VideoResolution::new(0, 36); - assert!(DemoVideoSource::new_blocking(config).is_err()); - - let mut config = test_config(); - config.framerate_fps = 0; - assert!(DemoVideoSource::new_blocking(config).is_err()); - } - - #[test] - fn yields_frames_with_configured_dimensions() { - if !gpu_available() { - eprintln!("skipping: no GPU adapter available"); - return; - } - let mut source = DemoVideoSource::new_blocking(test_config()).unwrap(); - - let first = source.next_frame(&PumpStop::new()).unwrap().unwrap(); - let second = source.next_frame(&PumpStop::new()).unwrap().unwrap(); - assert_eq!((first.buffer.width(), first.buffer.height()), (64, 36)); - assert_eq!(first.timestamp_us, 0); - assert_eq!(second.timestamp_us, 1_000); - } -} diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index aaa80776b..02259afdc 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -15,14 +15,11 @@ //! Ready-made capture sources. Each source is gated behind its own //! `source-*` feature. -#[cfg(feature = "source-demo")] -pub mod demo; - #[cfg(feature = "source-device")] pub mod device; #[cfg(feature = "source-gstreamer")] pub mod gstreamer; -#[cfg(feature = "source-shader")] -pub mod shader; +#[cfg(feature = "source-pattern")] +pub mod pattern; diff --git a/livekit-capture/src/sources/shader.rs b/livekit-capture/src/sources/pattern.rs similarity index 69% rename from livekit-capture/src/sources/shader.rs rename to livekit-capture/src/sources/pattern.rs index 3c6763d83..02493ff69 100644 --- a/livekit-capture/src/sources/shader.rs +++ b/livekit-capture/src/sources/pattern.rs @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Generative video from a WGSL shader. +//! Test pattern video source. //! -//! [`ShaderVideoSource`] renders each frame on the GPU with a WGSL shader -//! and yields the result as pixel video. The application supplies a -//! fragment snippet, inline or from a file ([`WgslShader`]). Rendering is +//! [`PatternVideoSource`] renders a built-in test pattern ([`Pattern`]) +//! on the GPU and yields the result as pixel video. Rendering is //! offscreen through [wgpu], so the source needs no window or display. //! //! The source reads each frame back from the GPU and converts it to I420 @@ -29,9 +28,7 @@ use crate::{ }; use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; use std::{ - borrow::Cow, fmt, - path::PathBuf, sync::{mpsc, Arc, Mutex}, thread, time::{Duration, Instant}, @@ -55,30 +52,17 @@ const STOP_POLL_INTERVAL: Duration = Duration::from_millis(100); /// Total time to wait for one frame readback before the source fails. const READBACK_TIMEOUT: Duration = Duration::from_secs(5); -/// Prelude prepended to a [`WgslShader::Fragment`] snippet. It draws one -/// triangle that covers the full target and calls `shade` per pixel. +/// Prelude prepended to every fragment snippet. It draws one triangle +/// that covers the full target and calls `shade` per pixel. const FRAGMENT_PRELUDE: &str = include_str!("../../shaders/prelude.wgsl"); -/// WGSL fragment snippet for a [`ShaderVideoSource`]. -/// -/// The snippet must define `fn shade(uv: vec2) -> vec4`. The -/// source calls `shade` once per pixel. `uv` runs from `(0.0, 0.0)` at -/// the top left to `(1.0, 1.0)` at the bottom right. The return value is -/// the pixel color as RGBA in the 0.0 to 1.0 range. Alpha is ignored. -/// -/// A fixed prelude declares these uniforms for the snippet: -/// -/// ```wgsl -/// struct LkUniforms { -/// resolution: vec2, // output size in pixels -/// time_s: f32, // seconds since the stream started -/// frame_index: u32, // frame counter, starts at 0 -/// } -/// @group(0) @binding(0) var lk: LkUniforms; -/// ``` +/// Fragment snippet for [`Pattern::Gradient`]. +const GRADIENT_SHADER: &str = include_str!("../../shaders/gradient.wgsl"); + +/// Test pattern rendered by a [`PatternVideoSource`]. /// -/// The prelude reserves the names `lk`, `LkUniforms`, `LkVertexOutput`, -/// `vs_main`, and `fs_main`. +/// Every pattern is a pure function of position, resolution, and time: +/// the same configuration produces the same frames on every machine. #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr( feature = "serde", @@ -87,33 +71,26 @@ const FRAGMENT_PRELUDE: &str = include_str!("../../shaders/prelude.wgsl"); )] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] #[non_exhaustive] -pub enum WgslShader { - /// Inline fragment snippet. - Fragment(String), - /// Path to a file that contains a fragment snippet. - /// - /// The source reads the file once, at construction. A relative path - /// resolves against the process working directory. - FragmentFile(PathBuf), +pub enum Pattern { + /// Animated color gradient. + Gradient, } -impl WgslShader { - /// Returns the complete WGSL module to compile, reading the snippet - /// from disk when needed. - fn module_code(&self) -> Result { - let snippet = match self { - Self::Fragment(snippet) => Cow::Borrowed(snippet.as_str()), - Self::FragmentFile(path) => { - Cow::Owned(std::fs::read_to_string(path).map_err(|error| { - ShaderVideoSourceError::ShaderFile { path: path.clone(), error } - })?) - } - }; - Ok(format!("{FRAGMENT_PRELUDE}\n{snippet}")) +impl Pattern { + /// Returns the complete WGSL module to compile. + fn module_code(&self) -> String { + match self { + Self::Gradient => assemble_module(GRADIENT_SHADER), + } } } -/// Configuration for a [`ShaderVideoSource`]. +/// Prepends the prelude to a pattern's fragment snippet. +fn assemble_module(snippet: &str) -> String { + format!("{FRAGMENT_PRELUDE}\n{snippet}") +} + +/// Configuration for a [`PatternVideoSource`]. #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr( feature = "serde", @@ -121,24 +98,24 @@ impl WgslShader { serde(deny_unknown_fields) )] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -pub struct ShaderVideoSourceConfig { +pub struct PatternVideoSourceConfig { /// Output resolution. pub resolution: VideoResolution, /// Output frame rate in frames per second. pub framerate_fps: u32, - /// Shader that colors each frame. - pub shader: WgslShader, + /// Pattern to render. + pub pattern: Pattern, } -/// Error returned by a shader video source. +/// Error returned by a pattern video source. #[derive(Debug, Error)] #[non_exhaustive] -pub enum ShaderVideoSourceError { +pub enum PatternVideoSourceError { /// The configured resolution has a zero component. - #[error("shader source resolution must be non-zero")] + #[error("pattern source resolution must be non-zero")] ZeroResolution, /// The configured frame rate is zero. - #[error("shader source frame rate must be non-zero")] + #[error("pattern source frame rate must be non-zero")] ZeroFramerate, /// No compatible GPU adapter is available. #[error("no compatible GPU adapter: {0}")] @@ -146,15 +123,6 @@ pub enum ShaderVideoSourceError { /// The GPU adapter rejected the device request. #[error("failed to open the GPU device: {0}")] Device(String), - /// The shader file could not be read. - #[error("failed to read shader file '{}': {error}", path.display())] - ShaderFile { - /// Path of the shader file. - path: PathBuf, - /// The underlying read error. - #[source] - error: std::io::Error, - }, /// The shader or its pipeline failed to build. #[error("failed to build the shader pipeline: {0}")] ShaderCompile(String), @@ -169,44 +137,41 @@ pub enum ShaderVideoSourceError { Convert(&'static str), } -/// Pixel video source that renders each frame on the GPU with a WGSL -/// shader. +/// Pixel video source that renders a test pattern on the GPU. /// -/// Construction selects a GPU adapter and compiles the shader, so a bad -/// shader fails construction, not the pump. The source sleeps to pace -/// itself to the configured frame rate. It never reaches the end of its -/// stream — stop the pump that drives it instead. -pub struct ShaderVideoSource { - config: ShaderVideoSourceConfig, - renderer: ShaderRenderer, +/// The source sleeps to pace itself to the configured frame rate. It +/// never reaches the end of its stream — stop the pump that drives it +/// instead. +pub struct PatternVideoSource { + config: PatternVideoSourceConfig, + renderer: PatternRenderer, started: Option, frame_index: u64, } -impl ShaderVideoSource { +impl PatternVideoSource { /// Creates the source. GPU setup runs on the tokio blocking pool. /// /// Requires a running tokio runtime. Use - /// [`ShaderVideoSource::new_blocking`] outside of async contexts. + /// [`PatternVideoSource::new_blocking`] outside of async contexts. #[cfg(feature = "tokio")] - pub async fn new(config: ShaderVideoSourceConfig) -> Result { + pub async fn new(config: PatternVideoSourceConfig) -> Result { crate::utils::run_blocking(move || Self::new_blocking(config)).await } - /// Selects a GPU adapter, compiles the shader, and builds the render - /// pipeline. + /// Selects a GPU adapter, compiles the pattern's shader, and builds + /// the render pipeline. /// - /// Construction fails when no GPU is available, when the shader does - /// not compile, when the shader file cannot be read, or for a zero + /// Construction fails when no GPU is available, or for a zero /// resolution or frame rate. - pub fn new_blocking(config: ShaderVideoSourceConfig) -> Result { + pub fn new_blocking(config: PatternVideoSourceConfig) -> Result { validate_config(&config).map_err(SourceError::new)?; - let renderer = ShaderRenderer::new(&config).map_err(SourceError::new)?; + let renderer = PatternRenderer::new(&config).map_err(SourceError::new)?; Ok(Self { config, renderer, started: None, frame_index: 0 }) } /// Returns the configuration the source was created with. - pub fn config(&self) -> &ShaderVideoSourceConfig { + pub fn config(&self) -> &PatternVideoSourceConfig { &self.config } @@ -215,7 +180,7 @@ impl ShaderVideoSource { } } -impl PixelVideoSource for ShaderVideoSource { +impl PixelVideoSource for PatternVideoSource { fn resolution(&self) -> VideoResolution { self.config.resolution } @@ -255,9 +220,9 @@ impl PixelVideoSource for ShaderVideoSource { } } -impl fmt::Debug for ShaderVideoSource { +impl fmt::Debug for PatternVideoSource { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ShaderVideoSource") + f.debug_struct("PatternVideoSource") .field("config", &self.config) .field("frame_index", &self.frame_index) .finish_non_exhaustive() @@ -265,19 +230,19 @@ impl fmt::Debug for ShaderVideoSource { } /// Validates the CPU-checkable parts of a configuration. -fn validate_config(config: &ShaderVideoSourceConfig) -> Result<(), ShaderVideoSourceError> { +fn validate_config(config: &PatternVideoSourceConfig) -> Result<(), PatternVideoSourceError> { let VideoResolution { width, height } = config.resolution; if width == 0 || height == 0 { - return Err(ShaderVideoSourceError::ZeroResolution); + return Err(PatternVideoSourceError::ZeroResolution); } if config.framerate_fps == 0 { - return Err(ShaderVideoSourceError::ZeroFramerate); + return Err(PatternVideoSourceError::ZeroFramerate); } Ok(()) } /// Owns the wgpu state and renders one frame at a time. -struct ShaderRenderer { +struct PatternRenderer { device: wgpu::Device, queue: wgpu::Queue, pipeline: wgpu::RenderPipeline, @@ -295,14 +260,13 @@ struct ShaderRenderer { device_error: Arc>>, } -impl ShaderRenderer { - fn new(config: &ShaderVideoSourceConfig) -> Result { +impl PatternRenderer { + fn new(config: &PatternVideoSourceConfig) -> Result { let VideoResolution { width, height } = config.resolution; - // Read and assemble the shader first, so a missing file fails - // before any GPU setup. - let module_code = config.shader.module_code()?; - let padded_bytes_per_row = padded_bytes_per_row(width) - .ok_or_else(|| ShaderVideoSourceError::Backend("resolution is too large".to_owned()))?; + let module_code = config.pattern.module_code(); + let padded_bytes_per_row = padded_bytes_per_row(width).ok_or_else(|| { + PatternVideoSourceError::Backend("resolution is too large".to_owned()) + })?; // Rendering is offscreen, so no display handle is needed. WGPU_* // environment variables can override the backend and adapter @@ -313,27 +277,27 @@ impl ShaderRenderer { power_preference: wgpu::PowerPreference::from_env().unwrap_or_default(), ..Default::default() })) - .map_err(|err| ShaderVideoSourceError::NoAdapter(err.to_string()))?; + .map_err(|err| PatternVideoSourceError::NoAdapter(err.to_string()))?; let info = adapter.get_info(); - log::info!("Rendering shader source on \"{}\" ({})", info.name, info.backend); + log::info!("Rendering pattern source on \"{}\" ({})", info.name, info.backend); // Clamp the default limits to what the adapter supports, so weaker // adapters (GL, software rasterizers) still open. A resolution // beyond the clamped limits fails texture creation below. let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { - label: Some("lk_shader_device"), + label: Some("lk_pattern_device"), required_limits: wgpu::Limits::default().or_worse_values_from(&adapter.limits()), ..Default::default() })) - .map_err(|err| ShaderVideoSourceError::Device(err.to_string()))?; + .map_err(|err| PatternVideoSourceError::Device(err.to_string()))?; // Runtime GPU errors have no return channel of their own: stash // the first one and report it from the next render_frame call. let device_error: Arc>> = Arc::default(); let sink = Arc::clone(&device_error); device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { - log::error!("shader source GPU error: {error}"); + log::error!("pattern source GPU error: {error}"); let mut slot = sink.lock().unwrap(); if slot.is_none() { *slot = Some(error.to_string()); @@ -344,12 +308,12 @@ impl ShaderRenderer { // so a bad shader fails construction with its compile message. let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("lk_shader_module"), + label: Some("lk_pattern_module"), source: wgpu::ShaderSource::Wgsl(module_code.into()), }); let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("lk_shader_bind_group_layout"), + label: Some("lk_pattern_bind_group_layout"), entries: &[wgpu::BindGroupLayoutEntry { binding: 0, visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, @@ -362,12 +326,12 @@ impl ShaderRenderer { }], }); let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("lk_shader_pipeline_layout"), + label: Some("lk_pattern_pipeline_layout"), bind_group_layouts: &[Some(&bind_group_layout)], immediate_size: 0, }); let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("lk_shader_pipeline"), + label: Some("lk_pattern_pipeline"), layout: Some(&pipeline_layout), vertex: wgpu::VertexState { module: &module, @@ -392,14 +356,14 @@ impl ShaderRenderer { cache: None, }); if let Some(error) = pollster::block_on(scope.pop()) { - return Err(ShaderVideoSourceError::ShaderCompile(error.to_string())); + return Err(PatternVideoSourceError::ShaderCompile(error.to_string())); } // Build the target and readback resources under their own scope, // so an unsupported resolution also fails construction. let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); let target = device.create_texture(&wgpu::TextureDescriptor { - label: Some("lk_shader_target"), + label: Some("lk_pattern_target"), size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, mip_level_count: 1, sample_count: 1, @@ -410,13 +374,13 @@ impl ShaderRenderer { }); let target_view = target.create_view(&wgpu::TextureViewDescriptor::default()); let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { - label: Some("lk_shader_uniforms"), + label: Some("lk_pattern_uniforms"), size: UNIFORM_BUFFER_SIZE, usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, mapped_at_creation: false, }); let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("lk_shader_bind_group"), + label: Some("lk_pattern_bind_group"), layout: &bind_group_layout, entries: &[wgpu::BindGroupEntry { binding: 0, @@ -424,13 +388,13 @@ impl ShaderRenderer { }], }); let staging = device.create_buffer(&wgpu::BufferDescriptor { - label: Some("lk_shader_staging"), + label: Some("lk_pattern_staging"), size: u64::from(padded_bytes_per_row) * u64::from(height), usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, mapped_at_creation: false, }); if let Some(error) = pollster::block_on(scope.pop()) { - return Err(ShaderVideoSourceError::Backend(error.to_string())); + return Err(PatternVideoSourceError::Backend(error.to_string())); } Ok(Self { @@ -455,7 +419,7 @@ impl ShaderRenderer { time_s: f32, frame_index: u32, stop: &PumpStop, - ) -> Result, ShaderVideoSourceError> { + ) -> Result, PatternVideoSourceError> { self.check_device_error()?; let VideoResolution { width, height } = self.resolution; @@ -467,10 +431,10 @@ impl ShaderRenderer { let mut encoder = self .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("lk_shader") }); + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("lk_pattern") }); { let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("lk_shader_pass"), + label: Some("lk_pattern_pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { view: &self.target_view, depth_slice: None, @@ -532,7 +496,7 @@ impl ShaderRenderer { submission: wgpu::SubmissionIndex, mapped: &mpsc::Receiver>, stop: &PumpStop, - ) -> Result { + ) -> Result { let deadline = Instant::now() + READBACK_TIMEOUT; loop { let poll = self.device.poll(wgpu::PollType::Wait { @@ -541,14 +505,14 @@ impl ShaderRenderer { }); match poll { Ok(_) | Err(wgpu::PollError::Timeout) => {} - Err(err) => return Err(ShaderVideoSourceError::Readback(err.to_string())), + Err(err) => return Err(PatternVideoSourceError::Readback(err.to_string())), } match mapped.try_recv() { Ok(Ok(())) => return Ok(true), - Ok(Err(err)) => return Err(ShaderVideoSourceError::Readback(err.to_string())), + Ok(Err(err)) => return Err(PatternVideoSourceError::Readback(err.to_string())), Err(mpsc::TryRecvError::Empty) => {} Err(mpsc::TryRecvError::Disconnected) => { - return Err(ShaderVideoSourceError::Readback( + return Err(PatternVideoSourceError::Readback( "map callback was dropped".to_owned(), )); } @@ -558,7 +522,7 @@ impl ShaderRenderer { return Ok(false); } if Instant::now() >= deadline { - return Err(ShaderVideoSourceError::Readback( + return Err(PatternVideoSourceError::Readback( "timed out waiting for the GPU".to_owned(), )); } @@ -566,9 +530,9 @@ impl ShaderRenderer { } /// Reports the first stashed GPU error, if there is one. - fn check_device_error(&self) -> Result<(), ShaderVideoSourceError> { + fn check_device_error(&self) -> Result<(), PatternVideoSourceError> { match &*self.device_error.lock().unwrap() { - Some(message) => Err(ShaderVideoSourceError::Backend(message.clone())), + Some(message) => Err(PatternVideoSourceError::Backend(message.clone())), None => Ok(()), } } @@ -611,16 +575,16 @@ fn convert_to_i420( source_stride: u32, width: u32, height: u32, -) -> Result { +) -> Result { if source.len() < source_stride as usize * height as usize { - return Err(ShaderVideoSourceError::Convert("mapped frame is too short")); + return Err(PatternVideoSourceError::Convert("mapped frame is too short")); } let source_stride = i32::try_from(source_stride) - .map_err(|_| ShaderVideoSourceError::Convert("stride exceeds supported range"))?; + .map_err(|_| PatternVideoSourceError::Convert("stride exceeds supported range"))?; let width_i32 = i32::try_from(width) - .map_err(|_| ShaderVideoSourceError::Convert("width exceeds supported range"))?; + .map_err(|_| PatternVideoSourceError::Convert("width exceeds supported range"))?; let height_i32 = i32::try_from(height) - .map_err(|_| ShaderVideoSourceError::Convert("height exceeds supported range"))?; + .map_err(|_| PatternVideoSourceError::Convert("height exceeds supported range"))?; let mut buffer = I420Buffer::new(width, height); let (stride_y, stride_u, stride_v) = buffer.strides(); @@ -643,7 +607,7 @@ fn convert_to_i420( ) }; if ret != 0 { - return Err(ShaderVideoSourceError::Convert("ARGBToI420 failed")); + return Err(PatternVideoSourceError::Convert("ARGBToI420 failed")); } Ok(buffer) } @@ -654,56 +618,34 @@ mod tests { const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; - fn red_config() -> ShaderVideoSourceConfig { - ShaderVideoSourceConfig { + fn gradient_config() -> PatternVideoSourceConfig { + PatternVideoSourceConfig { resolution: RESOLUTION, - framerate_fps: 200, - shader: WgslShader::Fragment( - "fn shade(uv: vec2) -> vec4 { return vec4(1.0, 0.0, 0.0, 1.0); }" - .to_owned(), - ), + framerate_fps: 1000, + pattern: Pattern::Gradient, } } #[test] fn validation_rejects_zero_resolution_and_framerate() { - let mut config = red_config(); + let mut config = gradient_config(); config.resolution = VideoResolution::new(0, 36); assert!(matches!( validate_config(&config), - Err(ShaderVideoSourceError::ZeroResolution) + Err(PatternVideoSourceError::ZeroResolution) )); - let mut config = red_config(); + let mut config = gradient_config(); config.framerate_fps = 0; - assert!(matches!(validate_config(&config), Err(ShaderVideoSourceError::ZeroFramerate))); + assert!(matches!(validate_config(&config), Err(PatternVideoSourceError::ZeroFramerate))); } #[test] - fn fragment_snippets_are_appended_to_the_prelude() { - let shader = WgslShader::Fragment("fn shade(...) {}".to_owned()); - let code = shader.module_code().unwrap(); + fn gradient_module_includes_the_prelude() { + let code = Pattern::Gradient.module_code(); assert!(code.contains("fn vs_main")); assert!(code.contains("fn fs_main")); - assert!(code.ends_with("fn shade(...) {}")); - } - - #[test] - fn fragment_files_are_read_from_disk() { - let path = std::env::temp_dir().join(format!("lk_shader_test_{}.wgsl", std::process::id())); - std::fs::write(&path, "fn shade(...) {}").unwrap(); - let code = WgslShader::FragmentFile(path.clone()).module_code(); - std::fs::remove_file(&path).ok(); - assert!(code.unwrap().ends_with("fn shade(...) {}")); - } - - #[test] - fn missing_fragment_files_fail() { - let shader = WgslShader::FragmentFile(PathBuf::from("lk_shader_test_missing.wgsl")); - assert!(matches!( - shader.module_code(), - Err(ShaderVideoSourceError::ShaderFile { .. }) - )); + assert!(code.contains("fn shade")); } #[test] @@ -715,36 +657,28 @@ mod tests { } #[test] - fn renders_solid_color_frames_at_the_frame_rate() { + fn gradient_renders_frames_at_the_frame_rate() { if !gpu_available() { eprintln!("skipping: no GPU adapter available"); return; } - let mut source = ShaderVideoSource::new_blocking(red_config()).unwrap(); + let mut source = PatternVideoSource::new_blocking(gradient_config()).unwrap(); let stop = PumpStop::new(); let first = source.next_frame(&stop).unwrap().unwrap(); let second = source.next_frame(&stop).unwrap().unwrap(); assert_eq!((first.buffer.width(), first.buffer.height()), (64, 36)); assert_eq!(first.timestamp_us, 0); - assert_eq!(second.timestamp_us, 5_000); + assert_eq!(second.timestamp_us, 1_000); - // Red in limited-range BT.601, as converted by libyuv. - let i420 = first.buffer.as_i420().expect("shader source yields I420 buffers"); + // The gradient's top-left pixel at time zero is red-dominant: + // RGB (255, 68, 47), which is about (121, 91, 211) in + // limited-range BT.601. A red/blue channel swap in the readback + // path flips the two chroma values, so this check catches it. + let i420 = first.buffer.as_i420().expect("pattern source yields I420 buffers"); let (y, u, v) = i420.data(); - assert!(y[0].abs_diff(82) <= 2, "unexpected luma {}", y[0]); - assert!(u[0].abs_diff(90) <= 2, "unexpected chroma-u {}", u[0]); - assert!(v[0].abs_diff(240) <= 2, "unexpected chroma-v {}", v[0]); - } - - #[test] - fn invalid_shaders_fail_construction() { - if !gpu_available() { - eprintln!("skipping: no GPU adapter available"); - return; - } - let mut config = red_config(); - config.shader = WgslShader::Fragment("this is not wgsl".to_owned()); - assert!(ShaderVideoSource::new_blocking(config).is_err()); + assert!(y[0].abs_diff(121) <= 5, "unexpected luma {}", y[0]); + assert!(u[0].abs_diff(91) <= 6, "unexpected chroma-u {}", u[0]); + assert!(v[0].abs_diff(211) <= 6, "unexpected chroma-v {}", v[0]); } } diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index c399032a6..709540a08 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -24,9 +24,9 @@ tracing = ["tokio/tracing", "console-subscriber"] # GStreamer. capture = [ "dep:livekit-capture", - "livekit-capture/source-demo", "livekit-capture/source-device", "livekit-capture/source-gstreamer", + "livekit-capture/source-pattern", "livekit-capture/tokio", ] diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 5f34835c9..96bf84e15 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -72,12 +72,20 @@ message GstreamerVideoSourceConfig { optional GstreamerRateControl rate_control = 4; } -// Test source rendering an animated color gradient on the GPU. -message DemoVideoSourceConfig { +// Test patterns built into livekit-capture. +enum Pattern { + // Animated color gradient. + PATTERN_GRADIENT = 0; +} + +// Test pattern rendered on the GPU. +message PatternVideoSourceConfig { // Output resolution. required VideoSourceResolution resolution = 1; // Output frame rate in frames per second. required uint32 framerate_fps = 2; + // Pattern to render. + required Pattern pattern = 3; } // Frame format delivered by a capture device. @@ -209,7 +217,7 @@ message OwnedCaptureSource { message NewCaptureSourceRequest { oneof config { GstreamerVideoSourceConfig gstreamer = 1; - DemoVideoSourceConfig demo = 2; + PatternVideoSourceConfig pattern = 2; DeviceVideoSourceConfig device = 4; } optional uint64 request_async_id = 3; diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index 9e8c35205..fb686bc94 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -17,12 +17,12 @@ use livekit_capture::{ encoded::EncodedVideoCodec, primitive::VideoResolution, sources::{ - demo::DemoVideoSourceConfig, device::{ DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, DeviceSelector, DeviceVideoSourceConfig, }, gstreamer::{GStreamerBitrateUnit, GStreamerRateControlConfig, GStreamerVideoSourceConfig}, + pattern::{Pattern, PatternVideoSourceConfig}, }, }; @@ -32,10 +32,18 @@ impl From for VideoResolution { } } -impl From for DemoVideoSourceConfig { - fn from(config: proto::DemoVideoSourceConfig) -> Self { - Self { resolution: config.resolution.into(), framerate_fps: config.framerate_fps } - } +pub fn pattern_config_from_proto( + config: proto::PatternVideoSourceConfig, +) -> FfiResult { + let pattern = proto::Pattern::try_from(config.pattern) + .map_err(|_| FfiError::InvalidRequest("invalid pattern".into()))?; + Ok(PatternVideoSourceConfig { + resolution: config.resolution.into(), + framerate_fps: config.framerate_fps, + pattern: match pattern { + proto::Pattern::Gradient => Pattern::Gradient, + }, + }) } impl From for GStreamerBitrateUnit { diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index 13c6dd768..44ec60321 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -24,9 +24,9 @@ use livekit_capture::{ pixel::{PixelVideoPump, PixelVideoSource}, pump::{PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, sources::{ - demo::DemoVideoSource, device::{self, DeviceVideoSource}, gstreamer::GStreamerVideoSource, + pattern::PatternVideoSource, }, }; use parking_lot::Mutex; @@ -35,7 +35,7 @@ use super::{video_source::FfiVideoSource, FfiHandle, FfiServer}; use crate::{ conversion::capture::{ device_config_from_proto, device_info_to_proto, gstreamer_config_from_proto, - video_codec_to_proto, + pattern_config_from_proto, video_codec_to_proto, }, proto, FfiError, FfiHandleId, FfiResult, }; @@ -119,8 +119,8 @@ async fn create_capture_source( let source: Box = Box::new(source); CapturePump::Encoded(EncodedVideoPump::new(source)) } - proto::new_capture_source_request::Config::Demo(config) => { - let source = DemoVideoSource::new(config.into()) + proto::new_capture_source_request::Config::Pattern(config) => { + let source = PatternVideoSource::new(pattern_config_from_proto(config)?) .await .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; let source: Box = Box::new(source); @@ -333,12 +333,13 @@ mod tests { } #[test] - fn demo_capture_lifecycle() { + fn pattern_capture_lifecycle() { let request = proto::NewCaptureSourceRequest { - config: Some(proto::new_capture_source_request::Config::Demo( - proto::DemoVideoSourceConfig { + config: Some(proto::new_capture_source_request::Config::Pattern( + proto::PatternVideoSourceConfig { resolution: proto::VideoSourceResolution { width: 1280, height: 720 }, framerate_fps: 30, + pattern: proto::Pattern::Gradient.into(), }, )), request_async_id: None, @@ -346,7 +347,7 @@ mod tests { let source = server() .async_runtime .block_on(create_capture_source(server(), request)) - .expect("demo capture source should build"); + .expect("pattern capture source should build"); assert_eq!(source.info.kind(), proto::CaptureSourceKind::CaptureSourcePixel); assert_eq!(source.info.resolution.width, 1280); let capture_handle = source.handle.id; From 1f74545d12527c8417e8155b091ae6e57fb01692 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:22:28 -0700 Subject: [PATCH 55/56] Logo pattern --- livekit-capture/shaders/logo.wgsl | 101 +++++++++++++++++++++++++ livekit-capture/src/sources/pattern.rs | 60 +++++++++++++-- livekit-ffi/protocol/capture.proto | 2 + livekit-ffi/src/conversion/capture.rs | 1 + 4 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 livekit-capture/shaders/logo.wgsl diff --git a/livekit-capture/shaders/logo.wgsl b/livekit-capture/shaders/logo.wgsl new file mode 100644 index 000000000..1d6f40073 --- /dev/null +++ b/livekit-capture/shaders/logo.wgsl @@ -0,0 +1,101 @@ +// Bouncing LiveKit logo: the built-in logo pattern. +// +// A white 7x7-cell tile carries the LiveKit glyph in black at its +// center. The tile moves in a straight line and reflects off the frame +// edges. Position is a pure function of time. +// +// All rectangles get about two pixels of edge feather. The soft edges +// make sub-pixel motion smooth, and they survive chroma subsampling and +// video encoding. + +// The glyph as a 5x5 bitmap. Each row is a 5-bit mask. The highest bit +// is the leftmost column: +// +// 1 0 0 0 1 +// 1 0 0 1 0 +// 1 0 1 0 0 +// 1 0 0 1 0 +// 1 1 1 0 1 +const GLYPH_ROWS: array = array(0x11u, 0x12u, 0x14u, 0x12u, 0x1du); + +// Cells per tile side, and the glyph offset into the tile, in cells. +const TILE_CELLS: f32 = 7.0; +const GLYPH_OFFSET: f32 = 1.0; + +// Tile height as a fraction of the frame height. This keeps the logo +// the same visual size at every resolution. +const LOGO_SIZE: f32 = 0.25; + +// Speed along each axis, in frame heights per second. The two values +// have no small common multiple, so the bounce path repeats slowly. +const SPEED: vec2 = vec2(0.23, 0.17); + +// Starting phase, so the logo does not start in a corner. +const START_PHASE: vec2 = vec2(0.34, 0.71); + +// White and black keep every edge luma-only. Luma has full resolution +// in 4:2:0 video, so these edges encode cleanly. +const TILE_COLOR: vec3 = vec3(1.0, 1.0, 1.0); +const GLYPH_COLOR: vec3 = vec3(0.0, 0.0, 0.0); +const BACKGROUND: vec3 = vec3(0.0, 0.0, 0.0); + +// Folds a growing phase into ping-pong motion between 0.0 and 1.0. +fn ping_pong(phase: f32) -> f32 { + return 1.0 - abs(1.0 - 2.0 * fract(phase * 0.5)); +} + +// Coverage of the rectangle [min_p, max_p] at point `p`. The interior +// is fully opaque. Alpha falls to zero over `feather` outside the edge, +// so touching rectangles union without seams. +fn rect_alpha(p: vec2, min_p: vec2, max_p: vec2, feather: f32) -> f32 { + let center = (min_p + max_p) * 0.5; + let half_size = (max_p - min_p) * 0.5; + let d = abs(p - center) - half_size; + let outside = length(max(d, vec2(0.0))); + let inside = min(max(d.x, d.y), 0.0); + return 1.0 - smoothstep(0.0, feather, outside + inside); +} + +fn shade(uv: vec2) -> vec4 { + // Work in a space that is `aspect` wide and 1.0 tall, so the cells + // stay square. Pixels in this space are 1.0 / height on both axes. + let height = max(lk.resolution.y, 1.0); + let aspect = lk.resolution.x / height; + let p = vec2(uv.x * aspect, uv.y); + + // Distance the logo can travel along each axis. The lower bound + // keeps the math finite when the frame is narrower than the logo. + let travel = max(vec2(aspect, 1.0) - vec2(LOGO_SIZE), vec2(0.0001)); + let phase = START_PHASE + lk.time_s * SPEED / travel; + let origin = vec2(ping_pong(phase.x), ping_pong(phase.y)) * travel; + + // About two output pixels of edge feather. + let feather = 2.0 / height; + + // Skip the coverage math outside the tile and its feather band. + let local = p - origin; + if local.x < -feather || local.x > LOGO_SIZE + feather + || local.y < -feather || local.y > LOGO_SIZE + feather { + return vec4(BACKGROUND, 1.0); + } + + // Coverage of the tile, and of the glyph cells inside it. + let tile = rect_alpha(p, origin, origin + vec2(LOGO_SIZE), feather); + + let cell_size = LOGO_SIZE / TILE_CELLS; + var glyph = 0.0; + for (var row = 0u; row < 5u; row = row + 1u) { + let bits = GLYPH_ROWS[row]; + for (var col = 0u; col < 5u; col = col + 1u) { + if ((bits >> (4u - col)) & 1u) == 0u { + continue; + } + let cell_min = origin + + (vec2(f32(col), f32(row)) + vec2(GLYPH_OFFSET)) * cell_size; + glyph = max(glyph, rect_alpha(p, cell_min, cell_min + vec2(cell_size), feather)); + } + } + + let logo = mix(TILE_COLOR, GLYPH_COLOR, glyph); + return vec4(mix(BACKGROUND, logo, tile), 1.0); +} diff --git a/livekit-capture/src/sources/pattern.rs b/livekit-capture/src/sources/pattern.rs index 02493ff69..64eab5e00 100644 --- a/livekit-capture/src/sources/pattern.rs +++ b/livekit-capture/src/sources/pattern.rs @@ -52,6 +52,14 @@ const STOP_POLL_INTERVAL: Duration = Duration::from_millis(100); /// Total time to wait for one frame readback before the source fails. const READBACK_TIMEOUT: Duration = Duration::from_secs(5); +/// Period after which the shader time uniform wraps: 2^13 seconds, +/// about 2.3 hours. +/// +/// f32 seconds lose precision as they grow. The wrap keeps the time +/// resolution finer than one millisecond on long runs, at the cost of +/// one pattern discontinuity per period. Frame timestamps do not wrap. +const TIME_WRAP_PERIOD_US: u64 = 8_192_000_000; + /// Prelude prepended to every fragment snippet. It draws one triangle /// that covers the full target and calls `shade` per pixel. const FRAGMENT_PRELUDE: &str = include_str!("../../shaders/prelude.wgsl"); @@ -59,6 +67,9 @@ const FRAGMENT_PRELUDE: &str = include_str!("../../shaders/prelude.wgsl"); /// Fragment snippet for [`Pattern::Gradient`]. const GRADIENT_SHADER: &str = include_str!("../../shaders/gradient.wgsl"); +/// Fragment snippet for [`Pattern::Logo`]. +const LOGO_SHADER: &str = include_str!("../../shaders/logo.wgsl"); + /// Test pattern rendered by a [`PatternVideoSource`]. /// /// Every pattern is a pure function of position, resolution, and time: @@ -74,6 +85,8 @@ const GRADIENT_SHADER: &str = include_str!("../../shaders/gradient.wgsl"); pub enum Pattern { /// Animated color gradient. Gradient, + /// Bouncing LiveKit logo. + Logo, } impl Pattern { @@ -81,6 +94,7 @@ impl Pattern { fn module_code(&self) -> String { match self { Self::Gradient => assemble_module(GRADIENT_SHADER), + Self::Logo => assemble_module(LOGO_SHADER), } } } @@ -193,18 +207,21 @@ impl PixelVideoSource for PatternVideoSource { // Pace against the ideal timeline so timestamps stay jitter-free. let interval_us = self.frame_interval().as_micros() as u64; - let elapsed = Duration::from_micros(self.frame_index.saturating_mul(interval_us)); + let elapsed_us = self.frame_index.saturating_mul(interval_us); + let elapsed = Duration::from_micros(elapsed_us); let due = started + elapsed; if let Some(wait) = due.checked_duration_since(Instant::now()) { thread::sleep(wait); } - // The uniform frame index wraps after u32::MAX frames. + // The shader time wraps to keep its f32 precision on long runs, + // and the uniform frame index wraps after u32::MAX frames. + let time_s = Duration::from_micros(elapsed_us % TIME_WRAP_PERIOD_US).as_secs_f32(); let frame_index = self.frame_index as u32; self.frame_index += 1; let buffer = self .renderer - .render_frame(elapsed.as_secs_f32(), frame_index, stop) + .render_frame(time_s, frame_index, stop) .map_err(SourceError::new)?; let Some(buffer) = buffer else { // The stop token fired during the readback wait. @@ -641,11 +658,13 @@ mod tests { } #[test] - fn gradient_module_includes_the_prelude() { - let code = Pattern::Gradient.module_code(); - assert!(code.contains("fn vs_main")); - assert!(code.contains("fn fs_main")); - assert!(code.contains("fn shade")); + fn pattern_modules_include_the_prelude() { + for pattern in [Pattern::Gradient, Pattern::Logo] { + let code = pattern.module_code(); + assert!(code.contains("fn vs_main"), "{pattern:?} is missing the prelude"); + assert!(code.contains("fn fs_main"), "{pattern:?} is missing the prelude"); + assert!(code.contains("fn shade"), "{pattern:?} is missing a shade function"); + } } #[test] @@ -681,4 +700,29 @@ mod tests { assert!(u[0].abs_diff(91) <= 6, "unexpected chroma-u {}", u[0]); assert!(v[0].abs_diff(211) <= 6, "unexpected chroma-v {}", v[0]); } + + #[test] + fn logo_renders_on_a_black_background() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut source = PatternVideoSource::new_blocking(PatternVideoSourceConfig { + resolution: RESOLUTION, + framerate_fps: 1000, + pattern: Pattern::Logo, + }) + .unwrap(); + + let frame = source.next_frame(&PumpStop::new()).unwrap().unwrap(); + let i420 = frame.buffer.as_i420().expect("pattern source yields I420 buffers"); + let (y, _, _) = i420.data(); + + // The logo starts away from the corners, so the top-left pixel is + // background black (luma 16 in limited range), and the lit logo + // cells stand out well above it. + assert!(y[0] <= 20, "top-left pixel is not background: {}", y[0]); + let lit = y.iter().filter(|&&luma| luma > 60).count(); + assert!(lit > 5, "no logo pixels found (lit count {lit})"); + } } diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 96bf84e15..9cb471ab4 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -76,6 +76,8 @@ message GstreamerVideoSourceConfig { enum Pattern { // Animated color gradient. PATTERN_GRADIENT = 0; + // Bouncing LiveKit logo. + PATTERN_LOGO = 1; } // Test pattern rendered on the GPU. diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index fb686bc94..c2eed4d77 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -42,6 +42,7 @@ pub fn pattern_config_from_proto( framerate_fps: config.framerate_fps, pattern: match pattern { proto::Pattern::Gradient => Pattern::Gradient, + proto::Pattern::Logo => Pattern::Logo, }, }) } From d3387cd52bbb42891ec47fd52066e0d500206083 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:56:44 -0700 Subject: [PATCH 56/56] Clock source Also extract render infra shared with pattern source. --- Cargo.lock | 1 + livekit-capture/Cargo.toml | 2 + livekit-capture/README.md | 1 + livekit-capture/shaders/clock.wgsl | 259 +++++++++++++ livekit-capture/src/encoded/h26x.rs | 10 +- livekit-capture/src/encoded/mod.rs | 1 - livekit-capture/src/error.rs | 3 + livekit-capture/src/lib.rs | 3 + livekit-capture/src/renderer.rs | 485 +++++++++++++++++++++++++ livekit-capture/src/sources/clock.rs | 272 ++++++++++++++ livekit-capture/src/sources/mod.rs | 3 + livekit-capture/src/sources/pattern.rs | 474 ++---------------------- livekit-ffi/Cargo.toml | 1 + livekit-ffi/protocol/capture.proto | 10 + livekit-ffi/src/conversion/capture.rs | 7 + livekit-ffi/src/server/capture.rs | 20 +- 16 files changed, 1099 insertions(+), 453 deletions(-) create mode 100644 livekit-capture/shaders/clock.wgsl create mode 100644 livekit-capture/src/renderer.rs create mode 100644 livekit-capture/src/sources/clock.rs diff --git a/Cargo.lock b/Cargo.lock index 0569e8c72..30408f71c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4160,6 +4160,7 @@ name = "livekit-capture" version = "0.1.0" dependencies = [ "bytes", + "chrono", "dispatch2", "gstreamer", "gstreamer-app", diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 8ad337ca1..6d89997b2 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true [dependencies] bytes = { workspace = true } +chrono = { version = "0.4", default-features = false, features = ["clock"], optional = true } gstreamer = { version = "0.25.2", optional = true } gstreamer-app = { version = "0.25.2", optional = true } livekit = { workspace = true } @@ -34,6 +35,7 @@ schemars = ["dep:schemars", "serde"] tokio = ["tokio/rt"] # Pixel sources +source-clock = ["dep:chrono", "dep:pollster", "dep:wgpu", "dep:yuv-sys"] source-device = [ "dep:yuv-sys", # macOS backend diff --git a/livekit-capture/README.md b/livekit-capture/README.md index 26d065525..63e7bbc22 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -62,3 +62,4 @@ named `source-`. Each module documents its source. | `source-device` | `DeviceVideoSource` | pixel | | `source-gstreamer` | `GStreamerVideoSource` | encoded | | `source-pattern` | `PatternVideoSource` | pixel | +| `source-clock` | `ClockVideoSource` | pixel | diff --git a/livekit-capture/shaders/clock.wgsl b/livekit-capture/shaders/clock.wgsl new file mode 100644 index 000000000..4d2cfb923 --- /dev/null +++ b/livekit-capture/shaders/clock.wgsl @@ -0,0 +1,259 @@ +// Wall clock: the shader for the clock video source. +// +// The clock shows HH:MM:SS.mmm as seven-segment digits, with a grid of +// cells below it. Each grid row fills to show one millisecond digit, so +// a viewer can read sub-frame time from a paused frame. +// +// The CPU samples the wall clock once per frame and sends the twelve +// character codes in the uniform. Codes 0 to 9 are digits, 10 is a +// colon, and 11 is a dot. +// +// Shapes get about 1.5 output pixels of edge feather, so they stay +// clean through video encoding. + +struct ClockUniform { + viewport_size: vec2, + _pad0: vec2, + chars0: vec4, + chars1: vec4, + chars2: vec4, +} + +@group(0) @binding(0) var clock: ClockUniform; + +struct VertexOut { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +const CHAR_COUNT: u32 = 12u; +const COLON_CODE: u32 = 10u; +const DOT_CODE: u32 = 11u; + +// Layout metrics, in layout units. The digits sit in one row, and the +// millisecond grid sits below them. +const DIGIT_HEIGHT: f32 = 1.85; +const CELL_WIDTH: f32 = 1.0; +const SEGMENT_THICKNESS: f32 = 0.16; +const COLON_WIDTH: f32 = 0.34; +const DOT_WIDTH: f32 = 0.24; +const GAP: f32 = 0.14; +const TOTAL_WIDTH: f32 = 9.0 * CELL_WIDTH + 2.0 * COLON_WIDTH + DOT_WIDTH + 11.0 * GAP; +const GRID_COLUMNS: u32 = 9u; +const GRID_ROWS: u32 = 3u; +const GRID_CELL: f32 = 0.72; +const GRID_COLUMN_GAP: f32 = 0.30; +const GRID_ROW_GAP: f32 = 0.30; +const GRID_TOP_GAP: f32 = 0.22; +const GRID_WIDTH: f32 = 9.0 * GRID_CELL + 8.0 * GRID_COLUMN_GAP; +const GRID_HEIGHT: f32 = 3.0 * GRID_CELL + 2.0 * GRID_ROW_GAP; +const GROUP_HEIGHT: f32 = DIGIT_HEIGHT + GRID_TOP_GAP + GRID_HEIGHT; + +// Warm white for lit shapes, dark gray for unfilled grid cells. +const FOREGROUND: vec3 = vec3(1.0, 0.98, 0.92); +const EMPTY_CELL: vec3 = vec3(0.14, 0.14, 0.14); + +// Segment masks for the digits 0 to 9. Bit `n` lights segment `n` of +// the seven-segment layout in segment_rect. +const DIGIT_MASKS: array = array( + 0x3fu, + 0x06u, + 0x5bu, + 0x4fu, + 0x66u, + 0x6du, + 0x7du, + 0x07u, + 0x7fu, + 0x6fu, +); + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOut { + // One triangle that covers the full target. `uv` runs from (0, 0) + // at the top left to (1, 1) at the bottom right. + let corner = vec2(f32((vertex_index << 1u) & 2u), f32(vertex_index & 2u)); + var out: VertexOut; + out.position = vec4(corner * 2.0 - 1.0, 0.0, 1.0); + out.uv = vec2(corner.x, 1.0 - corner.y); + return out; +} + +// Returns the character code at `index`, from the uniform. +fn char_at(index: u32) -> u32 { + if index < 4u { + return clock.chars0[index]; + } + if index < 8u { + return clock.chars1[index - 4u]; + } + return clock.chars2[index - 8u]; +} + +fn char_width(code: u32) -> f32 { + if code < 10u { + return CELL_WIDTH; + } + if code == COLON_CODE { + return COLON_WIDTH; + } + return DOT_WIDTH; +} + +// Bounds of one seven-segment segment, as (min_x, min_y, max_x, max_y). +fn segment_rect(segment: u32) -> vec4 { + let t = SEGMENT_THICKNESS; + let mid = DIGIT_HEIGHT * 0.5; + + switch segment { + case 0u: { return vec4(t, 0.0, CELL_WIDTH - t, t); } + case 1u: { return vec4(CELL_WIDTH - t, t, CELL_WIDTH, mid); } + case 2u: { return vec4(CELL_WIDTH - t, mid, CELL_WIDTH, DIGIT_HEIGHT - t); } + case 3u: { return vec4(t, DIGIT_HEIGHT - t, CELL_WIDTH - t, DIGIT_HEIGHT); } + case 4u: { return vec4(0.0, mid, t, DIGIT_HEIGHT - t); } + case 5u: { return vec4(0.0, t, t, mid); } + case 6u: { return vec4(t, mid - t * 0.5, CELL_WIDTH - t, mid + t * 0.5); } + default: { return vec4(0.0); } + } +} + +// Coverage of the rectangle [min_p, max_p] at point `p`. The interior +// is fully opaque. Alpha falls to zero over `feather` outside the edge, +// so touching rectangles union without seams. +fn rect_alpha(p: vec2, min_p: vec2, max_p: vec2, feather: f32) -> f32 { + let center = (min_p + max_p) * 0.5; + let half_size = (max_p - min_p) * 0.5; + let d = abs(p - center) - half_size; + let outside = length(max(d, vec2(0.0))); + let inside = min(max(d.x, d.y), 0.0); + return 1.0 - smoothstep(0.0, feather, outside + inside); +} + +fn circle_alpha(p: vec2, center: vec2, radius: f32, feather: f32) -> f32 { + return 1.0 - smoothstep(0.0, feather, length(p - center) - radius); +} + +// Coverage of one seven-segment digit with its origin at `origin`. +fn digit_alpha(p: vec2, origin: vec2, digit: u32, feather: f32) -> f32 { + if digit > 9u { + return 0.0; + } + + let local = p - origin; + let mask = DIGIT_MASKS[digit]; + var alpha = 0.0; + + for (var segment = 0u; segment < 7u; segment = segment + 1u) { + if (mask & (1u << segment)) != 0u { + let r = segment_rect(segment); + alpha = max(alpha, rect_alpha(local, r.xy, r.zw, feather)); + } + } + + return alpha; +} + +// Coverage of a colon or dot separator with its origin at `origin`. +fn separator_alpha(p: vec2, origin: vec2, code: u32, feather: f32) -> f32 { + let local = p - origin; + let center_x = char_width(code) * 0.5; + + if code == COLON_CODE { + let r = 0.095; + let top = circle_alpha(local, vec2(center_x, DIGIT_HEIGHT * 0.38), r, feather); + let bottom = circle_alpha(local, vec2(center_x, DIGIT_HEIGHT * 0.62), r, feather); + return max(top, bottom); + } + + if code == DOT_CODE { + return circle_alpha(local, vec2(center_x, DIGIT_HEIGHT - 0.095), 0.08, feather); + } + + return 0.0; +} + +// Coverage of the twelve clock characters. +fn chars_alpha(p: vec2, feather: f32) -> f32 { + // Skip the character loop outside the digit row. + if p.x < -feather || p.x > TOTAL_WIDTH + feather + || p.y < -feather || p.y > DIGIT_HEIGHT + feather { + return 0.0; + } + + var cursor = 0.0; + var alpha = 0.0; + + for (var index = 0u; index < CHAR_COUNT; index = index + 1u) { + let code = char_at(index); + let origin = vec2(cursor, 0.0); + + if code < 10u { + alpha = max(alpha, digit_alpha(p, origin, code, feather)); + } else { + alpha = max(alpha, separator_alpha(p, origin, code, feather)); + } + + cursor = cursor + char_width(code) + GAP; + } + + return alpha; +} + +// Coverage of the millisecond grid: filled cells in x, unfilled cells +// in y. Each row fills to show one millisecond digit. +fn grid_alpha(p: vec2, feather: f32) -> vec2 { + let grid_origin = + vec2((TOTAL_WIDTH - GRID_WIDTH) * 0.5, DIGIT_HEIGHT + GRID_TOP_GAP); + + // Skip the cell loop outside the grid. + let local = p - grid_origin; + if local.x < -feather || local.x > GRID_WIDTH + feather + || local.y < -feather || local.y > GRID_HEIGHT + feather { + return vec2(0.0, 0.0); + } + + var filled = 0.0; + var unfilled = 0.0; + + for (var row = 0u; row < GRID_ROWS; row = row + 1u) { + let row_digit = char_at(9u + row); + for (var column = 0u; column < GRID_COLUMNS; column = column + 1u) { + let cell_origin = grid_origin + vec2( + f32(column) * (GRID_CELL + GRID_COLUMN_GAP), + f32(row) * (GRID_CELL + GRID_ROW_GAP), + ); + let cell_alpha = rect_alpha(p, cell_origin, cell_origin + vec2(GRID_CELL), feather); + if column < row_digit { + filled = max(filled, cell_alpha); + } else { + unfilled = max(unfilled, cell_alpha); + } + } + } + + return vec2(filled, unfilled); +} + +@fragment +fn fs_main(in: VertexOut) -> @location(0) vec4 { + // Work in a space that is `aspect` wide and 1.0 tall. + let height = max(clock.viewport_size.y, 1.0); + let aspect = max(clock.viewport_size.x / height, 0.1); + let p = vec2(in.uv.x * aspect, in.uv.y); + + // Fit the clock into the frame with a margin, and center it. + let scale = min((aspect * 0.94) / TOTAL_WIDTH, 0.82 / GROUP_HEIGHT); + let scaled_size = vec2(TOTAL_WIDTH, GROUP_HEIGHT) * scale; + let origin = vec2((aspect - scaled_size.x) * 0.5, (1.0 - scaled_size.y) * 0.5); + let local_p = (p - origin) / scale; + + // About 1.5 output pixels of edge feather, in layout units. + let feather = 1.5 / (height * scale); + + let chars = chars_alpha(local_p, feather); + let grid = grid_alpha(local_p, feather); + let alpha = max(chars, grid.x); + + let color = max(EMPTY_CELL * grid.y, FOREGROUND * alpha); + return vec4(color, 1.0); +} diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index a14b86710..2790a0db4 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -143,7 +143,10 @@ impl AnnexBAccessUnitParser { self.drain_next(true) } - fn drain_next(&mut self, at_eof: bool) -> Result, H26xParseError> { + fn drain_next( + &mut self, + at_eof: bool, + ) -> Result, H26xParseError> { self.scan_pending(); if let Some(split_at) = @@ -267,7 +270,10 @@ impl AvcAccessUnitParser { self.drain_next(true) } - fn drain_next(&mut self, at_eof: bool) -> Result, H26xParseError> { + fn drain_next( + &mut self, + at_eof: bool, + ) -> Result, H26xParseError> { self.scan_pending(at_eof)?; if let Some(split_at) = avc_access_unit_split_index( diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs index 872865c2d..5d696a77f 100644 --- a/livekit-capture/src/encoded/mod.rs +++ b/livekit-capture/src/encoded/mod.rs @@ -196,4 +196,3 @@ impl EncodedVideoSource for Box { const _: () = { fn _assert_object_safe(_: &dyn EncodedVideoSource) {} }; - diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs index be0991726..93803d83e 100644 --- a/livekit-capture/src/error.rs +++ b/livekit-capture/src/error.rs @@ -20,6 +20,9 @@ use std::{error::Error as StdError, fmt}; +#[cfg(any(feature = "source-clock", feature = "source-pattern"))] +pub use crate::renderer::RendererError; + /// Error returned by a capture source. /// /// `Display` and [`StdError::source`] delegate to the wrapped backend diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs index c9c5b526f..b1aef1220 100644 --- a/livekit-capture/src/lib.rs +++ b/livekit-capture/src/lib.rs @@ -25,4 +25,7 @@ pub mod pixel; pub mod primitive; pub mod pump; pub mod sources; + +#[cfg(any(feature = "source-clock", feature = "source-pattern"))] +mod renderer; mod utils; diff --git a/livekit-capture/src/renderer.rs b/livekit-capture/src/renderer.rs new file mode 100644 index 000000000..a0b8da706 --- /dev/null +++ b/livekit-capture/src/renderer.rs @@ -0,0 +1,485 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Crate-internal GPU renderer shared by the shader-backed sources. +//! +//! [`ShaderRenderer`] renders a WGSL module offscreen through wgpu, one +//! frame at a time, and reads each frame back as I420. The caller +//! supplies the module and the per-frame uniform bytes. [`FramePacer`] +//! paces the frames against an ideal timeline. + +use crate::{primitive::VideoResolution, pump::PumpStop}; +use livekit::webrtc::video_frame::I420Buffer; +use std::{ + sync::{mpsc, Arc, Mutex}, + thread, + time::{Duration, Instant}, +}; +use thiserror::Error; + +/// Render target format. Its memory layout (B, G, R, A) is the layout +/// libyuv names ARGB. +const TARGET_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Bgra8Unorm; + +/// Bytes per pixel of [`TARGET_FORMAT`]. +const TARGET_BYTES_PER_PIXEL: u32 = 4; + +/// Upper bound on one blocking GPU wait, so the stop token is observed +/// promptly. +const STOP_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Total time to wait for one frame readback before the renderer fails. +const READBACK_TIMEOUT: Duration = Duration::from_secs(5); + +/// Error returned by the GPU renderer. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum RendererError { + /// No compatible GPU adapter is available. + #[error("no compatible GPU adapter: {0}")] + NoAdapter(String), + /// The GPU adapter rejected the device request. + #[error("failed to open the GPU device: {0}")] + Device(String), + /// The shader or its pipeline failed to build. + #[error("failed to build the shader pipeline: {0}")] + ShaderCompile(String), + /// The GPU reported an error. + #[error("GPU error: {0}")] + Backend(String), + /// Reading the rendered frame back from the GPU failed. + #[error("failed to read the frame back from the GPU: {0}")] + Readback(String), + /// Pixel conversion failed. + #[error("failed to convert the rendered frame to I420: {0}")] + Convert(&'static str), +} + +/// Paces frames against an ideal timeline, so frame timestamps are +/// jitter-free. +#[derive(Debug)] +pub(crate) struct FramePacer { + interval_us: u64, + started: Option, + frame_index: u64, +} + +impl FramePacer { + /// Creates a pacer. The frame rate must be non-zero. + pub(crate) fn new(framerate_fps: u32) -> Self { + let interval_us = (Duration::from_secs(1) / framerate_fps).as_micros() as u64; + Self { interval_us, started: None, frame_index: 0 } + } + + /// Sleeps until the next frame is due. Returns the elapsed time on + /// the ideal timeline and the index of the frame. + /// + /// The sleep is at most one frame interval. + pub(crate) fn wait_for_next_frame(&mut self) -> (Duration, u64) { + let started = *self.started.get_or_insert_with(Instant::now); + let elapsed = Duration::from_micros(self.frame_index.saturating_mul(self.interval_us)); + let due = started + elapsed; + if let Some(wait) = due.checked_duration_since(Instant::now()) { + thread::sleep(wait); + } + let frame_index = self.frame_index; + self.frame_index += 1; + (elapsed, frame_index) + } +} + +/// Renders a WGSL module offscreen and reads frames back as I420. +/// +/// The module must define a vertex entry point `vs_main` and a fragment +/// entry point `fs_main`. The renderer draws one triangle, which must +/// cover the full target. The module can declare one uniform buffer at +/// group 0, binding 0. The caller supplies its bytes for each frame. +pub(crate) struct ShaderRenderer { + device: wgpu::Device, + queue: wgpu::Queue, + pipeline: wgpu::RenderPipeline, + bind_group: wgpu::BindGroup, + uniform_buffer: wgpu::Buffer, + uniform_size: u64, + target: wgpu::Texture, + target_view: wgpu::TextureView, + /// Readback destination, reused across frames. Rows are padded to + /// the wgpu copy alignment. + staging: wgpu::Buffer, + padded_bytes_per_row: u32, + resolution: VideoResolution, + /// First uncaptured GPU error, stashed by the device error handler + /// and surfaced on the next frame. + device_error: Arc>>, +} + +impl ShaderRenderer { + /// Opens a GPU device, compiles the module, and builds the pipeline + /// and readback resources. + pub(crate) fn new( + resolution: VideoResolution, + module_code: &str, + uniform_size: u64, + ) -> Result { + let VideoResolution { width, height } = resolution; + let padded_bytes_per_row = padded_bytes_per_row(width) + .ok_or_else(|| RendererError::Backend("resolution is too large".to_owned()))?; + + // Rendering is offscreen, so no display handle is needed. WGPU_* + // environment variables can override the backend and adapter + // selection. + let instance = + wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::from_env().unwrap_or_default(), + ..Default::default() + })) + .map_err(|err| RendererError::NoAdapter(err.to_string()))?; + + let info = adapter.get_info(); + log::info!("Rendering with GPU \"{}\" ({})", info.name, info.backend); + + // Clamp the default limits to what the adapter supports, so weaker + // adapters (GL, software rasterizers) still open. A resolution + // beyond the clamped limits fails texture creation below. + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { + label: Some("lk_render_device"), + required_limits: wgpu::Limits::default().or_worse_values_from(&adapter.limits()), + ..Default::default() + })) + .map_err(|err| RendererError::Device(err.to_string()))?; + + // Runtime GPU errors have no return channel of their own: stash + // the first one and report it from the next render_frame call. + let device_error: Arc>> = Arc::default(); + let sink = Arc::clone(&device_error); + device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { + log::error!("render GPU error: {error}"); + let mut slot = sink.lock().unwrap(); + if slot.is_none() { + *slot = Some(error.to_string()); + } + })); + + // Compile the shader and build the pipeline under an error scope, + // so a bad shader fails construction with its compile message. + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("lk_render_module"), + source: wgpu::ShaderSource::Wgsl(module_code.into()), + }); + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("lk_render_bind_group_layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu::BufferSize::new(uniform_size), + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("lk_render_pipeline_layout"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("lk_render_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: TARGET_FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + if let Some(error) = pollster::block_on(scope.pop()) { + return Err(RendererError::ShaderCompile(error.to_string())); + } + + // Build the target and readback resources under their own scope, + // so an unsupported resolution also fails construction. + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let target = device.create_texture(&wgpu::TextureDescriptor { + label: Some("lk_render_target"), + size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: TARGET_FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let target_view = target.create_view(&wgpu::TextureViewDescriptor::default()); + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("lk_render_uniforms"), + size: uniform_size, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("lk_render_bind_group"), + layout: &bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }], + }); + let staging = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("lk_render_staging"), + size: u64::from(padded_bytes_per_row) * u64::from(height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + if let Some(error) = pollster::block_on(scope.pop()) { + return Err(RendererError::Backend(error.to_string())); + } + + Ok(Self { + device, + queue, + pipeline, + bind_group, + uniform_buffer, + uniform_size, + target, + target_view, + staging, + padded_bytes_per_row, + resolution, + device_error, + }) + } + + /// Renders one frame with the given uniform bytes and reads it back + /// as I420. Returns `Ok(None)` when the stop token fires during the + /// readback wait. + /// + /// Every blocking wait is bounded by [`STOP_POLL_INTERVAL`], so the + /// stop token is observed promptly. + pub(crate) fn render_frame( + &self, + uniform: &[u8], + stop: &PumpStop, + ) -> Result, RendererError> { + debug_assert_eq!(uniform.len() as u64, self.uniform_size); + self.check_device_error()?; + + let VideoResolution { width, height } = self.resolution; + self.queue.write_buffer(&self.uniform_buffer, 0, uniform); + + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("lk_render") }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("lk_render_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.target_view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + ..Default::default() + }); + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &self.bind_group, &[]); + pass.draw(0..3, 0..1); + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.target, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &self.staging, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.padded_bytes_per_row), + rows_per_image: None, + }, + }, + wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + ); + // Schedule the mapping with the submission, so no separate + // map_async call is needed after submit. + let (mapped_tx, mapped_rx) = mpsc::channel(); + encoder.map_buffer_on_submit(&self.staging, wgpu::MapMode::Read, .., move |result| { + let _ = mapped_tx.send(result); + }); + let submission = self.queue.submit([encoder.finish()]); + + if !self.wait_for_map(submission, &mapped_rx, stop)? { + // Stopped: cancel the pending mapping to leave the buffer + // reusable. + self.staging.unmap(); + return Ok(None); + } + + let mapped = self.staging.slice(..).get_mapped_range(); + let converted = convert_to_i420(&mapped, self.padded_bytes_per_row, width, height); + drop(mapped); + self.staging.unmap(); + converted.map(Some) + } + + /// Waits for the staging buffer to be mapped. Returns `Ok(false)` when + /// the stop token fires first. + fn wait_for_map( + &self, + submission: wgpu::SubmissionIndex, + mapped: &mpsc::Receiver>, + stop: &PumpStop, + ) -> Result { + let deadline = Instant::now() + READBACK_TIMEOUT; + loop { + let poll = self.device.poll(wgpu::PollType::Wait { + submission_index: Some(submission.clone()), + timeout: Some(STOP_POLL_INTERVAL), + }); + match poll { + Ok(_) | Err(wgpu::PollError::Timeout) => {} + Err(err) => return Err(RendererError::Readback(err.to_string())), + } + match mapped.try_recv() { + Ok(Ok(())) => return Ok(true), + Ok(Err(err)) => return Err(RendererError::Readback(err.to_string())), + Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => { + return Err(RendererError::Readback("map callback was dropped".to_owned())); + } + } + self.check_device_error()?; + if stop.is_stopped() { + return Ok(false); + } + if Instant::now() >= deadline { + return Err(RendererError::Readback("timed out waiting for the GPU".to_owned())); + } + } + } + + /// Reports the first stashed GPU error, if there is one. + fn check_device_error(&self) -> Result<(), RendererError> { + match &*self.device_error.lock().unwrap() { + Some(message) => Err(RendererError::Backend(message.clone())), + None => Ok(()), + } + } +} + +/// Returns the staging-buffer row stride: the pixel row size rounded up +/// to the wgpu copy alignment. `None` when the value overflows `u32`. +fn padded_bytes_per_row(width: u32) -> Option { + let unpadded = u64::from(width) * u64::from(TARGET_BYTES_PER_PIXEL); + let align = u64::from(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT); + u32::try_from(unpadded.div_ceil(align) * align).ok() +} + +/// Returns whether a GPU adapter is available. Tests that need a GPU +/// skip when there is none. +#[cfg(test)] +pub(crate) fn gpu_available() -> bool { + let instance = + wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); + pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).is_ok() +} + +/// Converts one padded BGRA image to a freshly allocated I420 buffer. +fn convert_to_i420( + source: &[u8], + source_stride: u32, + width: u32, + height: u32, +) -> Result { + if source.len() < source_stride as usize * height as usize { + return Err(RendererError::Convert("mapped frame is too short")); + } + let source_stride = i32::try_from(source_stride) + .map_err(|_| RendererError::Convert("stride exceeds supported range"))?; + let width_i32 = i32::try_from(width) + .map_err(|_| RendererError::Convert("width exceeds supported range"))?; + let height_i32 = i32::try_from(height) + .map_err(|_| RendererError::Convert("height exceeds supported range"))?; + + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers `height` rows of `source_stride` bytes, and the + // destination planes come from a freshly allocated I420Buffer with matching width, + // height, and strides. + let ret = unsafe { + yuv_sys::rs_ARGBToI420( + source.as_ptr(), + source_stride, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + }; + if ret != 0 { + return Err(RendererError::Convert("ARGBToI420 failed")); + } + Ok(buffer) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rows_are_padded_to_the_copy_alignment() { + assert_eq!(padded_bytes_per_row(64), Some(256)); + assert_eq!(padded_bytes_per_row(321), Some(1536)); + assert_eq!(padded_bytes_per_row(1280), Some(5120)); + assert_eq!(padded_bytes_per_row(u32::MAX), None); + } + + #[test] + fn pacer_reports_the_ideal_timeline() { + let mut pacer = FramePacer::new(1000); + let (first_elapsed, first_index) = pacer.wait_for_next_frame(); + let (second_elapsed, second_index) = pacer.wait_for_next_frame(); + assert_eq!((first_elapsed.as_micros(), first_index), (0, 0)); + assert_eq!((second_elapsed.as_micros(), second_index), (1_000, 1)); + } +} diff --git a/livekit-capture/src/sources/clock.rs b/livekit-capture/src/sources/clock.rs new file mode 100644 index 000000000..7dce25dba --- /dev/null +++ b/livekit-capture/src/sources/clock.rs @@ -0,0 +1,272 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Wall-clock video source, for latency measurement. +//! +//! [`ClockVideoSource`] renders the local time as HH:MM:SS.mmm on the +//! GPU, with a grid below the digits that shows the milliseconds as +//! filled cells. The source samples the wall clock once per frame. +//! Rendering is offscreen through [wgpu], so the source needs no window +//! or display. +//! +//! The source reads each frame back from the GPU and converts it to I420 +//! on the CPU. +//! +//! [wgpu]: https://wgpu.rs + +use crate::{ + error::SourceError, + pixel::PixelVideoSource, + primitive::VideoResolution, + pump::PumpStop, + renderer::{FramePacer, RendererError, ShaderRenderer}, +}; +use chrono::Timelike; +use livekit::webrtc::video_frame::{BoxVideoFrame, VideoFrame, VideoRotation}; +use std::fmt; +use thiserror::Error; + +/// Complete WGSL module for the clock. +const CLOCK_SHADER: &str = include_str!("../../shaders/clock.wgsl"); + +/// Number of characters on the clock face: HH:MM:SS.mmm. +const CHAR_COUNT: usize = 12; + +/// Character codes for the separators. Codes 0 to 9 are digits. +const COLON: u32 = 10; +const DOT: u32 = 11; + +/// Size of the uniform block: `vec2` + padding + 3 * `vec4`. +const UNIFORM_BUFFER_SIZE: u64 = 64; + +/// Configuration for a [`ClockVideoSource`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct ClockVideoSourceConfig { + /// Output resolution. + pub resolution: VideoResolution, + /// Output frame rate in frames per second. + pub framerate_fps: u32, +} + +/// Error returned by a clock video source. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ClockVideoSourceError { + /// The configured resolution has a zero component. + #[error("clock source resolution must be non-zero")] + ZeroResolution, + /// The configured frame rate is zero. + #[error("clock source frame rate must be non-zero")] + ZeroFramerate, + /// The GPU renderer failed. + #[error(transparent)] + Render(#[from] RendererError), +} + +/// Pixel video source that renders a wall clock with millisecond +/// precision. +/// +/// The source sleeps to pace itself to the configured frame rate. It +/// never reaches the end of its stream — stop the pump that drives it +/// instead. +pub struct ClockVideoSource { + config: ClockVideoSourceConfig, + renderer: ShaderRenderer, + pacer: FramePacer, +} + +impl ClockVideoSource { + /// Creates the source. GPU setup runs on the tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`ClockVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: ClockVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await + } + + /// Selects a GPU adapter, compiles the clock shader, and builds the + /// render pipeline. + /// + /// Construction fails when no GPU is available, or for a zero + /// resolution or frame rate. + pub fn new_blocking(config: ClockVideoSourceConfig) -> Result { + validate_config(&config).map_err(SourceError::new)?; + let renderer = ShaderRenderer::new(config.resolution, CLOCK_SHADER, UNIFORM_BUFFER_SIZE) + .map_err(|error| SourceError::new(ClockVideoSourceError::Render(error)))?; + let pacer = FramePacer::new(config.framerate_fps); + Ok(Self { config, renderer, pacer }) + } + + /// Returns the configuration the source was created with. + pub fn config(&self) -> ClockVideoSourceConfig { + self.config + } +} + +impl PixelVideoSource for ClockVideoSource { + fn resolution(&self) -> VideoResolution { + self.config.resolution + } + + // The pacing sleep is at most one frame interval, and the renderer + // bounds every readback wait, so the stop token is observed promptly. + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + let (elapsed, _) = self.pacer.wait_for_next_frame(); + + // Sample the wall clock after the pacing sleep, so the shown + // time is as close as possible to the capture time. + let now = chrono::Local::now(); + let chars = + clock_chars(now.hour(), now.minute(), now.second(), now.nanosecond() / 1_000_000); + let uniform = uniform_bytes(self.config.resolution, &chars); + + let buffer = self + .renderer + .render_frame(&uniform, stop) + .map_err(|error| SourceError::new(ClockVideoSourceError::Render(error)))?; + let Some(buffer) = buffer else { + // The stop token fired during the readback wait. + return Ok(None); + }; + + Ok(Some(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: elapsed.as_micros() as i64, + frame_metadata: None, + buffer: Box::new(buffer), + })) + } +} + +impl fmt::Debug for ClockVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ClockVideoSource").field("config", &self.config).finish_non_exhaustive() + } +} + +/// Validates the CPU-checkable parts of a configuration. +fn validate_config(config: &ClockVideoSourceConfig) -> Result<(), ClockVideoSourceError> { + let VideoResolution { width, height } = config.resolution; + if width == 0 || height == 0 { + return Err(ClockVideoSourceError::ZeroResolution); + } + if config.framerate_fps == 0 { + return Err(ClockVideoSourceError::ZeroFramerate); + } + Ok(()) +} + +/// Returns the twelve character codes for HH:MM:SS.mmm. +fn clock_chars(hour: u32, minute: u32, second: u32, millisecond: u32) -> [u32; CHAR_COUNT] { + [ + (hour / 10) % 10, + hour % 10, + COLON, + (minute / 10) % 10, + minute % 10, + COLON, + (second / 10) % 10, + second % 10, + DOT, + (millisecond / 100) % 10, + (millisecond / 10) % 10, + millisecond % 10, + ] +} + +/// Serializes the uniform block: viewport size, padding, and the twelve +/// character codes at their 16-byte-aligned offset. +fn uniform_bytes( + resolution: VideoResolution, + chars: &[u32; CHAR_COUNT], +) -> [u8; UNIFORM_BUFFER_SIZE as usize] { + let mut bytes = [0u8; UNIFORM_BUFFER_SIZE as usize]; + bytes[0..4].copy_from_slice(&(resolution.width as f32).to_ne_bytes()); + bytes[4..8].copy_from_slice(&(resolution.height as f32).to_ne_bytes()); + for (index, code) in chars.iter().enumerate() { + let offset = 16 + index * 4; + bytes[offset..offset + 4].copy_from_slice(&code.to_ne_bytes()); + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::renderer::gpu_available; + + fn test_config() -> ClockVideoSourceConfig { + ClockVideoSourceConfig { + resolution: VideoResolution { width: 320, height: 180 }, + framerate_fps: 1000, + } + } + + #[test] + fn clock_chars_render_three_millisecond_digits() { + assert_eq!(clock_chars(12, 34, 56, 789), [1, 2, COLON, 3, 4, COLON, 5, 6, DOT, 7, 8, 9]); + } + + #[test] + fn validation_rejects_zero_resolution_and_framerate() { + let mut config = test_config(); + config.resolution = VideoResolution::new(0, 180); + assert!(matches!(validate_config(&config), Err(ClockVideoSourceError::ZeroResolution))); + + let mut config = test_config(); + config.framerate_fps = 0; + assert!(matches!(validate_config(&config), Err(ClockVideoSourceError::ZeroFramerate))); + } + + #[test] + fn uniform_places_chars_at_their_alignment() { + let chars = clock_chars(12, 34, 56, 789); + let bytes = uniform_bytes(VideoResolution::new(1280, 720), &chars); + assert_eq!(f32::from_ne_bytes(bytes[0..4].try_into().unwrap()), 1280.0); + assert_eq!(f32::from_ne_bytes(bytes[4..8].try_into().unwrap()), 720.0); + assert_eq!(u32::from_ne_bytes(bytes[16..20].try_into().unwrap()), 1); + assert_eq!(u32::from_ne_bytes(bytes[60..64].try_into().unwrap()), 9); + } + + #[test] + fn clock_renders_frames() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut source = ClockVideoSource::new_blocking(test_config()).unwrap(); + + let stop = PumpStop::new(); + let first = source.next_frame(&stop).unwrap().unwrap(); + let second = source.next_frame(&stop).unwrap().unwrap(); + assert_eq!((first.buffer.width(), first.buffer.height()), (320, 180)); + assert_eq!(first.timestamp_us, 0); + assert_eq!(second.timestamp_us, 1_000); + + // The clock is centered with a margin, so the top-left pixel is + // background black, and the lit digits stand out well above it. + let i420 = first.buffer.as_i420().expect("clock source yields I420 buffers"); + let (y, _, _) = i420.data(); + assert!(y[0] <= 20, "top-left pixel is not background: {}", y[0]); + let lit = y.iter().filter(|&&luma| luma > 100).count(); + assert!(lit > 20, "no clock pixels found (lit count {lit})"); + } +} diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index 02259afdc..eca6dc31a 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -15,6 +15,9 @@ //! Ready-made capture sources. Each source is gated behind its own //! `source-*` feature. +#[cfg(feature = "source-clock")] +pub mod clock; + #[cfg(feature = "source-device")] pub mod device; diff --git a/livekit-capture/src/sources/pattern.rs b/livekit-capture/src/sources/pattern.rs index 64eab5e00..1604f48d9 100644 --- a/livekit-capture/src/sources/pattern.rs +++ b/livekit-capture/src/sources/pattern.rs @@ -24,34 +24,19 @@ //! [wgpu]: https://wgpu.rs use crate::{ - error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop, -}; -use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; -use std::{ - fmt, - sync::{mpsc, Arc, Mutex}, - thread, - time::{Duration, Instant}, + error::SourceError, + pixel::PixelVideoSource, + primitive::VideoResolution, + pump::PumpStop, + renderer::{FramePacer, RendererError, ShaderRenderer}, }; +use livekit::webrtc::video_frame::{BoxVideoFrame, VideoFrame, VideoRotation}; +use std::{fmt, time::Duration}; use thiserror::Error; -/// Render target format. Its memory layout (B, G, R, A) is the layout -/// libyuv names ARGB. -const TARGET_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Bgra8Unorm; - -/// Bytes per pixel of [`TARGET_FORMAT`]. -const TARGET_BYTES_PER_PIXEL: u32 = 4; - /// Size of the uniform block: `vec2` + `f32` + `u32`. const UNIFORM_BUFFER_SIZE: u64 = 16; -/// Upper bound on one blocking GPU wait, so the stop token is observed -/// promptly. -const STOP_POLL_INTERVAL: Duration = Duration::from_millis(100); - -/// Total time to wait for one frame readback before the source fails. -const READBACK_TIMEOUT: Duration = Duration::from_secs(5); - /// Period after which the shader time uniform wraps: 2^13 seconds, /// about 2.3 hours. /// @@ -131,24 +116,9 @@ pub enum PatternVideoSourceError { /// The configured frame rate is zero. #[error("pattern source frame rate must be non-zero")] ZeroFramerate, - /// No compatible GPU adapter is available. - #[error("no compatible GPU adapter: {0}")] - NoAdapter(String), - /// The GPU adapter rejected the device request. - #[error("failed to open the GPU device: {0}")] - Device(String), - /// The shader or its pipeline failed to build. - #[error("failed to build the shader pipeline: {0}")] - ShaderCompile(String), - /// The GPU reported an error. - #[error("GPU error: {0}")] - Backend(String), - /// Reading the rendered frame back from the GPU failed. - #[error("failed to read the frame back from the GPU: {0}")] - Readback(String), - /// Pixel conversion failed. - #[error("failed to convert the rendered frame to I420: {0}")] - Convert(&'static str), + /// The GPU renderer failed. + #[error(transparent)] + Render(#[from] RendererError), } /// Pixel video source that renders a test pattern on the GPU. @@ -158,9 +128,8 @@ pub enum PatternVideoSourceError { /// instead. pub struct PatternVideoSource { config: PatternVideoSourceConfig, - renderer: PatternRenderer, - started: Option, - frame_index: u64, + renderer: ShaderRenderer, + pacer: FramePacer, } impl PatternVideoSource { @@ -180,18 +149,20 @@ impl PatternVideoSource { /// resolution or frame rate. pub fn new_blocking(config: PatternVideoSourceConfig) -> Result { validate_config(&config).map_err(SourceError::new)?; - let renderer = PatternRenderer::new(&config).map_err(SourceError::new)?; - Ok(Self { config, renderer, started: None, frame_index: 0 }) + let renderer = ShaderRenderer::new( + config.resolution, + &config.pattern.module_code(), + UNIFORM_BUFFER_SIZE, + ) + .map_err(|error| SourceError::new(PatternVideoSourceError::Render(error)))?; + let pacer = FramePacer::new(config.framerate_fps); + Ok(Self { config, renderer, pacer }) } /// Returns the configuration the source was created with. pub fn config(&self) -> &PatternVideoSourceConfig { &self.config } - - fn frame_interval(&self) -> Duration { - Duration::from_secs(1) / self.config.framerate_fps - } } impl PixelVideoSource for PatternVideoSource { @@ -199,30 +170,21 @@ impl PixelVideoSource for PatternVideoSource { self.config.resolution } - // The pacing sleep is at most one frame interval, and every readback - // wait is bounded by STOP_POLL_INTERVAL, so the stop token is - // observed promptly. + // The pacing sleep is at most one frame interval, and the renderer + // bounds every readback wait, so the stop token is observed promptly. fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { - let started = *self.started.get_or_insert_with(Instant::now); - - // Pace against the ideal timeline so timestamps stay jitter-free. - let interval_us = self.frame_interval().as_micros() as u64; - let elapsed_us = self.frame_index.saturating_mul(interval_us); - let elapsed = Duration::from_micros(elapsed_us); - let due = started + elapsed; - if let Some(wait) = due.checked_duration_since(Instant::now()) { - thread::sleep(wait); - } + let (elapsed, frame_index) = self.pacer.wait_for_next_frame(); // The shader time wraps to keep its f32 precision on long runs, // and the uniform frame index wraps after u32::MAX frames. + let elapsed_us = elapsed.as_micros() as u64; let time_s = Duration::from_micros(elapsed_us % TIME_WRAP_PERIOD_US).as_secs_f32(); - let frame_index = self.frame_index as u32; - self.frame_index += 1; + let uniform = uniform_bytes(self.config.resolution, time_s, frame_index as u32); + let buffer = self .renderer - .render_frame(time_s, frame_index, stop) - .map_err(SourceError::new)?; + .render_frame(&uniform, stop) + .map_err(|error| SourceError::new(PatternVideoSourceError::Render(error)))?; let Some(buffer) = buffer else { // The stop token fired during the readback wait. return Ok(None); @@ -239,10 +201,7 @@ impl PixelVideoSource for PatternVideoSource { impl fmt::Debug for PatternVideoSource { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PatternVideoSource") - .field("config", &self.config) - .field("frame_index", &self.frame_index) - .finish_non_exhaustive() + f.debug_struct("PatternVideoSource").field("config", &self.config).finish_non_exhaustive() } } @@ -258,303 +217,6 @@ fn validate_config(config: &PatternVideoSourceConfig) -> Result<(), PatternVideo Ok(()) } -/// Owns the wgpu state and renders one frame at a time. -struct PatternRenderer { - device: wgpu::Device, - queue: wgpu::Queue, - pipeline: wgpu::RenderPipeline, - bind_group: wgpu::BindGroup, - uniform_buffer: wgpu::Buffer, - target: wgpu::Texture, - target_view: wgpu::TextureView, - /// Readback destination, reused across frames. Rows are padded to - /// the wgpu copy alignment. - staging: wgpu::Buffer, - padded_bytes_per_row: u32, - resolution: VideoResolution, - /// First uncaptured GPU error, stashed by the device error handler - /// and surfaced on the next frame. - device_error: Arc>>, -} - -impl PatternRenderer { - fn new(config: &PatternVideoSourceConfig) -> Result { - let VideoResolution { width, height } = config.resolution; - let module_code = config.pattern.module_code(); - let padded_bytes_per_row = padded_bytes_per_row(width).ok_or_else(|| { - PatternVideoSourceError::Backend("resolution is too large".to_owned()) - })?; - - // Rendering is offscreen, so no display handle is needed. WGPU_* - // environment variables can override the backend and adapter - // selection. - let instance = - wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); - let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::from_env().unwrap_or_default(), - ..Default::default() - })) - .map_err(|err| PatternVideoSourceError::NoAdapter(err.to_string()))?; - - let info = adapter.get_info(); - log::info!("Rendering pattern source on \"{}\" ({})", info.name, info.backend); - - // Clamp the default limits to what the adapter supports, so weaker - // adapters (GL, software rasterizers) still open. A resolution - // beyond the clamped limits fails texture creation below. - let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { - label: Some("lk_pattern_device"), - required_limits: wgpu::Limits::default().or_worse_values_from(&adapter.limits()), - ..Default::default() - })) - .map_err(|err| PatternVideoSourceError::Device(err.to_string()))?; - - // Runtime GPU errors have no return channel of their own: stash - // the first one and report it from the next render_frame call. - let device_error: Arc>> = Arc::default(); - let sink = Arc::clone(&device_error); - device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { - log::error!("pattern source GPU error: {error}"); - let mut slot = sink.lock().unwrap(); - if slot.is_none() { - *slot = Some(error.to_string()); - } - })); - - // Compile the shader and build the pipeline under an error scope, - // so a bad shader fails construction with its compile message. - let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); - let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("lk_pattern_module"), - source: wgpu::ShaderSource::Wgsl(module_code.into()), - }); - let bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("lk_pattern_bind_group_layout"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, - ty: wgpu::BindingType::Buffer { - ty: wgpu::BufferBindingType::Uniform, - has_dynamic_offset: false, - min_binding_size: wgpu::BufferSize::new(UNIFORM_BUFFER_SIZE), - }, - count: None, - }], - }); - let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("lk_pattern_pipeline_layout"), - bind_group_layouts: &[Some(&bind_group_layout)], - immediate_size: 0, - }); - let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("lk_pattern_pipeline"), - layout: Some(&pipeline_layout), - vertex: wgpu::VertexState { - module: &module, - entry_point: Some("vs_main"), - compilation_options: Default::default(), - buffers: &[], - }, - primitive: wgpu::PrimitiveState::default(), - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - fragment: Some(wgpu::FragmentState { - module: &module, - entry_point: Some("fs_main"), - compilation_options: Default::default(), - targets: &[Some(wgpu::ColorTargetState { - format: TARGET_FORMAT, - blend: None, - write_mask: wgpu::ColorWrites::ALL, - })], - }), - multiview_mask: None, - cache: None, - }); - if let Some(error) = pollster::block_on(scope.pop()) { - return Err(PatternVideoSourceError::ShaderCompile(error.to_string())); - } - - // Build the target and readback resources under their own scope, - // so an unsupported resolution also fails construction. - let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); - let target = device.create_texture(&wgpu::TextureDescriptor { - label: Some("lk_pattern_target"), - size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: TARGET_FORMAT, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, - view_formats: &[], - }); - let target_view = target.create_view(&wgpu::TextureViewDescriptor::default()); - let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { - label: Some("lk_pattern_uniforms"), - size: UNIFORM_BUFFER_SIZE, - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - mapped_at_creation: false, - }); - let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("lk_pattern_bind_group"), - layout: &bind_group_layout, - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: uniform_buffer.as_entire_binding(), - }], - }); - let staging = device.create_buffer(&wgpu::BufferDescriptor { - label: Some("lk_pattern_staging"), - size: u64::from(padded_bytes_per_row) * u64::from(height), - usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, - mapped_at_creation: false, - }); - if let Some(error) = pollster::block_on(scope.pop()) { - return Err(PatternVideoSourceError::Backend(error.to_string())); - } - - Ok(Self { - device, - queue, - pipeline, - bind_group, - uniform_buffer, - target, - target_view, - staging, - padded_bytes_per_row, - resolution: config.resolution, - device_error, - }) - } - - /// Renders one frame and reads it back as I420. Returns `Ok(None)` - /// when the stop token fires during the readback wait. - fn render_frame( - &self, - time_s: f32, - frame_index: u32, - stop: &PumpStop, - ) -> Result, PatternVideoSourceError> { - self.check_device_error()?; - - let VideoResolution { width, height } = self.resolution; - self.queue.write_buffer( - &self.uniform_buffer, - 0, - &uniform_bytes(self.resolution, time_s, frame_index), - ); - - let mut encoder = self - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("lk_pattern") }); - { - let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("lk_pattern_pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &self.target_view, - depth_slice: None, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), - store: wgpu::StoreOp::Store, - }, - })], - ..Default::default() - }); - pass.set_pipeline(&self.pipeline); - pass.set_bind_group(0, &self.bind_group, &[]); - pass.draw(0..3, 0..1); - } - encoder.copy_texture_to_buffer( - wgpu::TexelCopyTextureInfo { - texture: &self.target, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::TexelCopyBufferInfo { - buffer: &self.staging, - layout: wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(self.padded_bytes_per_row), - rows_per_image: None, - }, - }, - wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, - ); - // Schedule the mapping with the submission, so no separate - // map_async call is needed after submit. - let (mapped_tx, mapped_rx) = mpsc::channel(); - encoder.map_buffer_on_submit(&self.staging, wgpu::MapMode::Read, .., move |result| { - let _ = mapped_tx.send(result); - }); - let submission = self.queue.submit([encoder.finish()]); - - if !self.wait_for_map(submission, &mapped_rx, stop)? { - // Stopped: cancel the pending mapping to leave the buffer - // reusable. - self.staging.unmap(); - return Ok(None); - } - - let mapped = self.staging.slice(..).get_mapped_range(); - let converted = convert_to_i420(&mapped, self.padded_bytes_per_row, width, height); - drop(mapped); - self.staging.unmap(); - converted.map(Some) - } - - /// Waits for the staging buffer to be mapped. Returns `Ok(false)` when - /// the stop token fires first. - fn wait_for_map( - &self, - submission: wgpu::SubmissionIndex, - mapped: &mpsc::Receiver>, - stop: &PumpStop, - ) -> Result { - let deadline = Instant::now() + READBACK_TIMEOUT; - loop { - let poll = self.device.poll(wgpu::PollType::Wait { - submission_index: Some(submission.clone()), - timeout: Some(STOP_POLL_INTERVAL), - }); - match poll { - Ok(_) | Err(wgpu::PollError::Timeout) => {} - Err(err) => return Err(PatternVideoSourceError::Readback(err.to_string())), - } - match mapped.try_recv() { - Ok(Ok(())) => return Ok(true), - Ok(Err(err)) => return Err(PatternVideoSourceError::Readback(err.to_string())), - Err(mpsc::TryRecvError::Empty) => {} - Err(mpsc::TryRecvError::Disconnected) => { - return Err(PatternVideoSourceError::Readback( - "map callback was dropped".to_owned(), - )); - } - } - self.check_device_error()?; - if stop.is_stopped() { - return Ok(false); - } - if Instant::now() >= deadline { - return Err(PatternVideoSourceError::Readback( - "timed out waiting for the GPU".to_owned(), - )); - } - } - } - - /// Reports the first stashed GPU error, if there is one. - fn check_device_error(&self) -> Result<(), PatternVideoSourceError> { - match &*self.device_error.lock().unwrap() { - Some(message) => Err(PatternVideoSourceError::Backend(message.clone())), - None => Ok(()), - } - } -} - /// Serializes the uniform block: resolution, time, and frame index. fn uniform_bytes( resolution: VideoResolution, @@ -569,69 +231,10 @@ fn uniform_bytes( bytes } -/// Returns the staging-buffer row stride: the pixel row size rounded up -/// to the wgpu copy alignment. `None` when the value overflows `u32`. -fn padded_bytes_per_row(width: u32) -> Option { - let unpadded = u64::from(width) * u64::from(TARGET_BYTES_PER_PIXEL); - let align = u64::from(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT); - u32::try_from(unpadded.div_ceil(align) * align).ok() -} - -/// Returns whether a GPU adapter is available. Tests that need a GPU -/// skip when there is none. -#[cfg(test)] -pub(crate) fn gpu_available() -> bool { - let instance = - wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); - pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).is_ok() -} - -/// Converts one padded BGRA image to a freshly allocated I420 buffer. -fn convert_to_i420( - source: &[u8], - source_stride: u32, - width: u32, - height: u32, -) -> Result { - if source.len() < source_stride as usize * height as usize { - return Err(PatternVideoSourceError::Convert("mapped frame is too short")); - } - let source_stride = i32::try_from(source_stride) - .map_err(|_| PatternVideoSourceError::Convert("stride exceeds supported range"))?; - let width_i32 = i32::try_from(width) - .map_err(|_| PatternVideoSourceError::Convert("width exceeds supported range"))?; - let height_i32 = i32::try_from(height) - .map_err(|_| PatternVideoSourceError::Convert("height exceeds supported range"))?; - - let mut buffer = I420Buffer::new(width, height); - let (stride_y, stride_u, stride_v) = buffer.strides(); - let (dst_y, dst_u, dst_v) = buffer.data_mut(); - // SAFETY: The source slice covers `height` rows of `source_stride` bytes, and the - // destination planes come from a freshly allocated I420Buffer with matching width, - // height, and strides. - let ret = unsafe { - yuv_sys::rs_ARGBToI420( - source.as_ptr(), - source_stride, - dst_y.as_mut_ptr(), - stride_y as i32, - dst_u.as_mut_ptr(), - stride_u as i32, - dst_v.as_mut_ptr(), - stride_v as i32, - width_i32, - height_i32, - ) - }; - if ret != 0 { - return Err(PatternVideoSourceError::Convert("ARGBToI420 failed")); - } - Ok(buffer) -} - #[cfg(test)] mod tests { use super::*; + use crate::renderer::gpu_available; const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; @@ -647,10 +250,7 @@ mod tests { fn validation_rejects_zero_resolution_and_framerate() { let mut config = gradient_config(); config.resolution = VideoResolution::new(0, 36); - assert!(matches!( - validate_config(&config), - Err(PatternVideoSourceError::ZeroResolution) - )); + assert!(matches!(validate_config(&config), Err(PatternVideoSourceError::ZeroResolution))); let mut config = gradient_config(); config.framerate_fps = 0; @@ -667,14 +267,6 @@ mod tests { } } - #[test] - fn rows_are_padded_to_the_copy_alignment() { - assert_eq!(padded_bytes_per_row(64), Some(256)); - assert_eq!(padded_bytes_per_row(321), Some(1536)); - assert_eq!(padded_bytes_per_row(1280), Some(5120)); - assert_eq!(padded_bytes_per_row(u32::MAX), None); - } - #[test] fn gradient_renders_frames_at_the_frame_rate() { if !gpu_available() { @@ -719,8 +311,8 @@ mod tests { let (y, _, _) = i420.data(); // The logo starts away from the corners, so the top-left pixel is - // background black (luma 16 in limited range), and the lit logo - // cells stand out well above it. + // background black (luma 16 in limited range), and the lit tile + // pixels stand out well above it. assert!(y[0] <= 20, "top-left pixel is not background: {}", y[0]); let lit = y.iter().filter(|&&luma| luma > 60).count(); assert!(lit > 5, "no logo pixels found (lit count {lit})"); diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index 709540a08..11216cde6 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -24,6 +24,7 @@ tracing = ["tokio/tracing", "console-subscriber"] # GStreamer. capture = [ "dep:livekit-capture", + "livekit-capture/source-clock", "livekit-capture/source-device", "livekit-capture/source-gstreamer", "livekit-capture/source-pattern", diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 9cb471ab4..4a5099e55 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -90,6 +90,15 @@ message PatternVideoSourceConfig { required Pattern pattern = 3; } +// Wall clock with millisecond precision, rendered on the GPU. Shows the +// local time of the machine that runs the FFI server. +message ClockVideoSourceConfig { + // Output resolution. + required VideoSourceResolution resolution = 1; + // Output frame rate in frames per second. + required uint32 framerate_fps = 2; +} + // Frame format delivered by a capture device. enum DeviceFrameFormat { DEVICE_FRAME_FORMAT_I420 = 0; @@ -221,6 +230,7 @@ message NewCaptureSourceRequest { GstreamerVideoSourceConfig gstreamer = 1; PatternVideoSourceConfig pattern = 2; DeviceVideoSourceConfig device = 4; + ClockVideoSourceConfig clock = 5; } optional uint64 request_async_id = 3; } diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index c2eed4d77..0d94b36ac 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -17,6 +17,7 @@ use livekit_capture::{ encoded::EncodedVideoCodec, primitive::VideoResolution, sources::{ + clock::ClockVideoSourceConfig, device::{ DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, DeviceSelector, DeviceVideoSourceConfig, @@ -32,6 +33,12 @@ impl From for VideoResolution { } } +impl From for ClockVideoSourceConfig { + fn from(config: proto::ClockVideoSourceConfig) -> Self { + Self { resolution: config.resolution.into(), framerate_fps: config.framerate_fps } + } +} + pub fn pattern_config_from_proto( config: proto::PatternVideoSourceConfig, ) -> FfiResult { diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index 44ec60321..1264f3627 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -24,6 +24,7 @@ use livekit_capture::{ pixel::{PixelVideoPump, PixelVideoSource}, pump::{PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, sources::{ + clock::ClockVideoSource, device::{self, DeviceVideoSource}, gstreamer::GStreamerVideoSource, pattern::PatternVideoSource, @@ -133,6 +134,13 @@ async fn create_capture_source( let source: Box = Box::new(source); CapturePump::Pixel(PixelVideoPump::new(source)) } + proto::new_capture_source_request::Config::Clock(config) => { + let source = ClockVideoSource::new(config.into()) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Pixel(PixelVideoPump::new(source)) + } }; let (kind, resolution, codec, publish_options, rtc_source, stop) = match &pump { @@ -189,10 +197,7 @@ async fn create_capture_source( }, ); - Ok(proto::OwnedCaptureSource { - handle: proto::FfiOwnedHandle { id: capture_handle_id }, - info, - }) + Ok(proto::OwnedCaptureSource { handle: proto::FfiOwnedHandle { id: capture_handle_id }, info }) } /// Maps the pump-derived publish options into the proto options the client @@ -354,11 +359,8 @@ mod tests { // Stopping before starting is allowed; the pump then exits // immediately once started, and the watcher marks it finished. - let response = on_stop_capture( - server(), - proto::StopCaptureRequest { capture_handle }, - ) - .unwrap(); + let response = + on_stop_capture(server(), proto::StopCaptureRequest { capture_handle }).unwrap(); assert_eq!(response.error, None); let response =