diff --git a/.changeset/livekit-capture.md b/.changeset/livekit-capture.md new file mode 100644 index 000000000..2ef99a76d --- /dev/null +++ b/.changeset/livekit-capture.md @@ -0,0 +1,5 @@ +--- +"livekit-capture": minor +--- + +Add a `livekit-capture` crate with codec-neutral encoded capture types, H264/H265/VP8/VP9/AV1 passthrough support, common encoded ingress helpers, and GStreamer appsink encoded ingress. Encoded sources honor WebRTC rate-control targets, validate pre-encoded AV1 and H265 access units on ingest, and support opt-in frame metadata for capture latency measurement. diff --git a/Cargo.lock b/Cargo.lock index 9b47a71a1..f01ff7dba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,6 +553,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomic_refcell" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" + [[package]] name = "autocfg" version = "1.5.1" @@ -572,7 +578,7 @@ dependencies = [ "log", "num-rational", "num-traits", - "pastey", + "pastey 0.1.1", "rayon", "thiserror 2.0.19", "v_frame", @@ -1896,6 +1902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.13.1", + "block2 0.6.2", "objc2 0.6.4", ] @@ -2694,8 +2701,21 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0071fe88dba8e40086c8ff9bbb62622999f49628344b1d1bf490a48a29d80f22" dependencies = [ - "glib-sys", - "gobject-sys", + "glib-sys 0.21.5", + "gobject-sys 0.21.5", + "libc", + "system-deps", + "windows-sys 0.61.2", +] + +[[package]] +name = "gio-sys" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353fdc7da7cd16da916104b1e0e4e7de380ec9c8aaa20d4d742d66310ab4b0d5" +dependencies = [ + "glib-sys 0.22.8", + "gobject-sys 0.22.6", "libc", "system-deps", "windows-sys 0.61.2", @@ -2724,10 +2744,31 @@ dependencies = [ "futures-executor", "futures-task", "futures-util", - "gio-sys", - "glib-macros", - "glib-sys", - "gobject-sys", + "gio-sys 0.21.5", + "glib-macros 0.21.5", + "glib-sys 0.21.5", + "gobject-sys 0.21.5", + "libc", + "memchr", + "smallvec", +] + +[[package]] +name = "glib" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddbcf514bd1881fc1b960e4e52b4e82873f4da3bceddbd58d42827b508888100" +dependencies = [ + "bitflags 2.13.1", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "futures-util", + "gio-sys 0.22.8", + "glib-macros 0.22.6", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", "libc", "memchr", "smallvec", @@ -2746,6 +2787,18 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "glib-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "506d23499707c7142898429757e8d9a3871d965239a2cb66dfa05052be6d6f19" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "glib-sys" version = "0.21.5" @@ -2756,6 +2809,16 @@ dependencies = [ "system-deps", ] +[[package]] +name = "glib-sys" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030967459f9f676851872c6304adea7825c6d462ec9b72554c733cf0c5952233" +dependencies = [ + "libc", + "system-deps", +] + [[package]] name = "glifo" version = "0.1.1" @@ -2817,7 +2880,18 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dca35da0d19a18f4575f3cb99fe1c9e029a2941af5662f326f738a21edaf294" dependencies = [ - "glib-sys", + "glib-sys 0.21.5", + "libc", + "system-deps", +] + +[[package]] +name = "gobject-sys" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22a861859b887a79cf461359c192c97a57d8fb0229dd291232e57aa11f6fa72c" +dependencies = [ + "glib-sys 0.22.8", "libc", "system-deps", ] @@ -2867,6 +2941,99 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "gstreamer" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4527e1b9bae8d29ce137bde5b8eec8ae8f78f13ad00fc6e70cbe227d6ad027" +dependencies = [ + "cfg-if 1.0.4", + "futures-channel", + "futures-core", + "futures-util", + "glib 0.22.8", + "gstreamer-sys", + "itertools 0.15.0", + "kstring", + "libc", + "muldiv", + "num-integer", + "num-rational", + "option-operations", + "pastey 0.2.3", + "pin-project-lite", + "smallvec", + "thiserror 2.0.19", +] + +[[package]] +name = "gstreamer-app" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97f8ae9238c2352398dcc084de28df3f7099af216ac6c160b52318d23f25c010" +dependencies = [ + "futures-core", + "futures-sink", + "glib 0.22.8", + "gstreamer", + "gstreamer-app-sys", + "gstreamer-base", + "libc", +] + +[[package]] +name = "gstreamer-app-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a74a8211e5d7df2f45b612c284ddf56b92bdf4e879e8ed72e7c46dd0842e158" +dependencies = [ + "glib-sys 0.22.8", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-base" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91c94a4d3047d05dd6e1f6d91c74f61f56384c7ea1c9d0c1051572eeeb0138d" +dependencies = [ + "atomic_refcell", + "cfg-if 1.0.4", + "glib 0.22.8", + "gstreamer", + "gstreamer-base-sys", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fbbc623dc066908ba10c43d629c21096508dea04796592a206c4edd864e37" +dependencies = [ + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533fa8d28fc830eafccbcfcfddb390563ea5d3a351af2c3aab99e197e5f5b1ba" +dependencies = [ + "cfg-if 1.0.4", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "libc", + "system-deps", +] + [[package]] name = "guillotiere" version = "0.7.0" @@ -3507,6 +3674,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3785,7 +3961,7 @@ version = "0.3.45" dependencies = [ "cxx", "env_logger 0.11.11", - "glib", + "glib 0.21.5", "jni 0.21.1", "js-sys", "lazy_static", @@ -3979,6 +4155,34 @@ dependencies = [ "url", ] +[[package]] +name = "livekit-capture" +version = "0.1.0" +dependencies = [ + "bytes", + "chrono", + "dispatch2", + "gstreamer", + "gstreamer-app", + "image", + "libc", + "livekit", + "log", + "objc2 0.6.4", + "objc2-av-foundation", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation 0.3.2", + "pollster", + "schemars", + "serde", + "thiserror 2.0.19", + "tokio", + "v4l", + "wgpu", + "yuv-sys", +] + [[package]] name = "livekit-common" version = "0.1.1" @@ -4048,6 +4252,7 @@ dependencies = [ "link-cplusplus", "livekit", "livekit-api", + "livekit-capture", "livekit-protocol", "log", "parking_lot", @@ -4387,6 +4592,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "muldiv" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0" + [[package]] name = "multimap" version = "0.10.1" @@ -4876,6 +5087,19 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2 0.6.4", + "objc2-core-media", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -4911,6 +5135,28 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.1", + "objc2 0.6.4", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -5001,6 +5247,21 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + [[package]] name = "objc2-core-text" version = "0.3.2" @@ -5013,6 +5274,19 @@ dependencies = [ "objc2-core-graphics", ] +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags 2.13.1", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", +] + [[package]] name = "objc2-encode" version = "4.1.0" @@ -5318,6 +5592,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-operations" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aca39cf52b03268400c16eeb9b56382ea3c3353409309b63f5c8f0b1faf42754" +dependencies = [ + "pastey 0.2.3", +] + [[package]] name = "orbclient" version = "0.3.55" @@ -5434,6 +5717,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pbjson" version = "0.6.0" @@ -6281,6 +6570,26 @@ dependencies = [ "bitflags 2.13.1", ] +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "regex" version = "1.13.1" @@ -6613,6 +6922,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.3", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -6758,6 +7092,17 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_json" version = "1.0.151" @@ -7851,7 +8196,7 @@ dependencies = [ "num-complex", "num-integer", "num-traits", - "pastey", + "pastey 0.1.1", "rustfft", "smallvec", "tract-data", @@ -7914,7 +8259,7 @@ dependencies = [ "liquid-derive", "log", "num-traits", - "pastey", + "pastey 0.1.1", "scan_fmt", "smallvec", "time", diff --git a/Cargo.toml b/Cargo.toml index f241cab17..7de999834 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "livekit-datatrack", "livekit-token-source", "livekit-ffi-node-bindings", + "livekit-capture", "livekit-net", "livekit-runtime", "livekit-wakeword", @@ -53,6 +54,7 @@ imgproc = { version = "0.3.19", path = "imgproc" } libwebrtc = { version = "0.3.45", path = "libwebrtc" } livekit = { version = "0.8.3", path = "livekit" } livekit-api = { version = "0.6.3", path = "livekit-api" } +livekit-capture = { version = "0.1.0", path = "livekit-capture" } livekit-ffi = { version = "0.12.75", path = "livekit-ffi" } livekit-datatrack = { version = "0.1.13", path = "livekit-datatrack" } livekit-token-source = { version = "0.1.1", path = "livekit-token-source" } @@ -85,6 +87,7 @@ prost = "0.14" prost-build = "0.14" prost-types = "0.14" rand = "0.9" +schemars = "1.2" serde = "1" serde_json = "1.0" thiserror = "2" diff --git a/livekit-capture/AGENTS.md b/livekit-capture/AGENTS.md new file mode 100644 index 000000000..19fa12189 --- /dev/null +++ b/livekit-capture/AGENTS.md @@ -0,0 +1,22 @@ +# AGENTS.md + +## Adding a source + +- Add a feature gate in `Cargo.toml` + - Name the feature after the source module with a `source-` prefix + (e.g., module `gstreamer` → feature `source-gstreamer`) + - If the new source requires dependencies, they should only be include when feature is enabled +- Define a module for the new source in `src/sources/mod.rs`: + ```rust + #[cfg(feature = "source-my-source")] + pub mod my_source; + ``` +- Follow conventions for existing sources +- Source implementation should be self-contained in its module and submodules + - Only hoist functionality to a higher level module when shared by multiple sources +- Source (e.g, `MyVideoSource`) is constructed from a config struct (e.g., `MyVideoSourceConfig`) +- Add conditional derives for `serde` and `schemars` for config +- Implement `PixelVideoSource` or `EncodedVideoSource` for your source + - NEVER add a source that is not consumable uniformly through one of these traits + - Doing so would break API contract and break integration for consumers +- Keep API surface minimal and hide implementation details diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml new file mode 100644 index 000000000..6d89997b2 --- /dev/null +++ b/livekit-capture/Cargo.toml @@ -0,0 +1,96 @@ +[package] +name = "livekit-capture" +description = "Encoded video ingest helpers for LiveKit" +version = "0.1.0" +readme = "README.md" +license.workspace = true +edition.workspace = true +repository.workspace = true + +[dependencies] +bytes = { workspace = true } +chrono = { version = "0.4", default-features = false, features = ["clock"], optional = true } +gstreamer = { version = "0.25.2", optional = true } +gstreamer-app = { version = "0.25.2", optional = true } +livekit = { workspace = true } +log = { workspace = true } +pollster = { version = "0.4", optional = true } +schemars = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"], optional = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["sync"] } +wgpu = { workspace = true, optional = true } +yuv-sys = { workspace = true, features = ["jpeg"], optional = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "time", "macros"] } + +[features] +default = [] +serde = ["dep:serde"] +schemars = ["dep:schemars", "serde"] + +# Async convenience wrappers for blocking source construction when using +# a tokio runtime. +tokio = ["tokio/rt"] + +# Pixel sources +source-clock = ["dep:chrono", "dep:pollster", "dep:wgpu", "dep:yuv-sys"] +source-device = [ + "dep:yuv-sys", + # macOS backend + "dep:dispatch2", + "dep:objc2", + "dep:objc2-av-foundation", + "dep:objc2-core-media", + "dep:objc2-core-video", + "dep:objc2-foundation", + "objc2-av-foundation/AVCaptureDevice", + "objc2-av-foundation/AVCaptureInput", + "objc2-av-foundation/AVCaptureOutputBase", + "objc2-av-foundation/AVCaptureSession", + "objc2-av-foundation/AVCaptureSessionPreset", + "objc2-av-foundation/AVCaptureVideoDataOutput", + "objc2-av-foundation/AVMediaFormat", + "objc2-av-foundation/AVVideoSettings", + "objc2-av-foundation/dispatch2", + "objc2-av-foundation/objc2-core-media", + "objc2-core-media/CMFormatDescription", + "objc2-core-media/CMSync", + "objc2-core-media/CMTime", + "objc2-core-media/CMSampleBuffer", + "objc2-core-media/objc2-core-video", + "objc2-core-video/CVBase", + "objc2-core-video/CVBuffer", + "objc2-core-video/CVImageBuffer", + "objc2-core-video/CVPixelBuffer", + "objc2-core-video/CVReturn", + "objc2-foundation/NSArray", + "objc2-foundation/NSDictionary", + "objc2-foundation/NSError", + "objc2-foundation/NSObject", + "objc2-foundation/NSValue", + "objc2-foundation/NSString", + "objc2-foundation/objc2-core-foundation", + # Linux backend + "dep:image", + "dep:libc", + "dep:v4l", +] +source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] + +# Encoded sources +source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] + +[target.'cfg(target_os = "macos")'.dependencies] +dispatch2 = { version = "0.3.1", default-features = false, features = ["std"], optional = true } +objc2 = { version = "0.6.4", default-features = false, features = ["std"], optional = true } +objc2-av-foundation = { version = "0.3.2", default-features = false, optional = true } +objc2-core-media = { version = "0.3.2", default-features = false, optional = true } +objc2-core-video = { version = "0.3.2", default-features = false, optional = true } +objc2-foundation = { version = "0.3.2", default-features = false, features = ["std"], optional = true } + +[target.'cfg(target_os = "linux")'.dependencies] +image = { workspace = true, optional = true } +libc = { version = "0.2", optional = true } +v4l = { version = "0.14", default-features = false, features = ["v4l2"], optional = true } diff --git a/livekit-capture/README.md b/livekit-capture/README.md new file mode 100644 index 000000000..63e7bbc22 --- /dev/null +++ b/livekit-capture/README.md @@ -0,0 +1,65 @@ +# LiveKit Capture + +> [!IMPORTANT] +> This crate is currently in Developer Preview mode and not ready for production use. +> There may be bugs, and APIs and configuration options are subject to change during this period. + +This crate provides video capture sources and the pumps that publish them +with the LiveKit [Rust SDK](../livekit/README.md). Pick a ready-made source, +or implement one small trait to add your own. The application runs and +supervises every source the same way. + +## Source and pump + +Two concepts make up the crate. Video reaches a LiveKit track in one of two +forms, so the source and the pump each have two variants. + +**A source** produces video, one blocking call at a time. Use a ready-made +source (see [Sources](#sources)), or implement the trait to add your own: + +- A `pixel::PixelVideoSource` produces raw `VideoFrame`s — from a camera, for + example. The WebRTC encoder encodes them. +- An `encoded::EncodedVideoSource` produces access units that are already + encoded — from an encoding pipeline, for example. The SDK sends them to the + wire without re-encoding (passthrough). When frames are encoded upstream, + this removes an extra decode and encode step and lowers latency. + +**A pump** connects one source to an RTC video source: +`pixel::PixelVideoPump` or `encoded::EncodedVideoPump`. It builds the +matching RTC video source, derives the publish options, and runs the capture +loop. Spawn a pump onto a dedicated thread and it becomes a +`pump::RunningPump`. Both pump kinds spawn into the same type, so an +application supervises them the same way. Stop a running pump from any +thread through its stop handle. + +Sources block, so the pumps run synchronous code on plain threads. + +## Publishing a track + +A pump supplies the RTC source and the publish options, so publication is the +same for either path. + +```rust +let pump = PixelVideoPump::new(PatternVideoSource::new(config).await?); + +let track = LocalVideoTrack::create_video_track("pattern", pump.rtc_source()); +let options = pump.publish_options(); +room.local_participant().publish_track(LocalTrack::Video(track), options).await?; + +let running = pump.spawn()?; + +// On shutdown: +let stats = running.stop_and_join_async().await?; +``` + +## Sources + +Each source lives in its own module under `sources`, behind a Cargo feature +named `source-`. Each module documents its source. + +| Feature | Source | Kind | +| ------------------ | ---------------------- | ------- | +| `source-device` | `DeviceVideoSource` | pixel | +| `source-gstreamer` | `GStreamerVideoSource` | encoded | +| `source-pattern` | `PatternVideoSource` | pixel | +| `source-clock` | `ClockVideoSource` | pixel | diff --git a/livekit-capture/shaders/clock.wgsl b/livekit-capture/shaders/clock.wgsl new file mode 100644 index 000000000..4d2cfb923 --- /dev/null +++ b/livekit-capture/shaders/clock.wgsl @@ -0,0 +1,259 @@ +// Wall clock: the shader for the clock video source. +// +// The clock shows HH:MM:SS.mmm as seven-segment digits, with a grid of +// cells below it. Each grid row fills to show one millisecond digit, so +// a viewer can read sub-frame time from a paused frame. +// +// The CPU samples the wall clock once per frame and sends the twelve +// character codes in the uniform. Codes 0 to 9 are digits, 10 is a +// colon, and 11 is a dot. +// +// Shapes get about 1.5 output pixels of edge feather, so they stay +// clean through video encoding. + +struct ClockUniform { + viewport_size: vec2, + _pad0: vec2, + chars0: vec4, + chars1: vec4, + chars2: vec4, +} + +@group(0) @binding(0) var clock: ClockUniform; + +struct VertexOut { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +const CHAR_COUNT: u32 = 12u; +const COLON_CODE: u32 = 10u; +const DOT_CODE: u32 = 11u; + +// Layout metrics, in layout units. The digits sit in one row, and the +// millisecond grid sits below them. +const DIGIT_HEIGHT: f32 = 1.85; +const CELL_WIDTH: f32 = 1.0; +const SEGMENT_THICKNESS: f32 = 0.16; +const COLON_WIDTH: f32 = 0.34; +const DOT_WIDTH: f32 = 0.24; +const GAP: f32 = 0.14; +const TOTAL_WIDTH: f32 = 9.0 * CELL_WIDTH + 2.0 * COLON_WIDTH + DOT_WIDTH + 11.0 * GAP; +const GRID_COLUMNS: u32 = 9u; +const GRID_ROWS: u32 = 3u; +const GRID_CELL: f32 = 0.72; +const GRID_COLUMN_GAP: f32 = 0.30; +const GRID_ROW_GAP: f32 = 0.30; +const GRID_TOP_GAP: f32 = 0.22; +const GRID_WIDTH: f32 = 9.0 * GRID_CELL + 8.0 * GRID_COLUMN_GAP; +const GRID_HEIGHT: f32 = 3.0 * GRID_CELL + 2.0 * GRID_ROW_GAP; +const GROUP_HEIGHT: f32 = DIGIT_HEIGHT + GRID_TOP_GAP + GRID_HEIGHT; + +// Warm white for lit shapes, dark gray for unfilled grid cells. +const FOREGROUND: vec3 = vec3(1.0, 0.98, 0.92); +const EMPTY_CELL: vec3 = vec3(0.14, 0.14, 0.14); + +// Segment masks for the digits 0 to 9. Bit `n` lights segment `n` of +// the seven-segment layout in segment_rect. +const DIGIT_MASKS: array = array( + 0x3fu, + 0x06u, + 0x5bu, + 0x4fu, + 0x66u, + 0x6du, + 0x7du, + 0x07u, + 0x7fu, + 0x6fu, +); + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOut { + // One triangle that covers the full target. `uv` runs from (0, 0) + // at the top left to (1, 1) at the bottom right. + let corner = vec2(f32((vertex_index << 1u) & 2u), f32(vertex_index & 2u)); + var out: VertexOut; + out.position = vec4(corner * 2.0 - 1.0, 0.0, 1.0); + out.uv = vec2(corner.x, 1.0 - corner.y); + return out; +} + +// Returns the character code at `index`, from the uniform. +fn char_at(index: u32) -> u32 { + if index < 4u { + return clock.chars0[index]; + } + if index < 8u { + return clock.chars1[index - 4u]; + } + return clock.chars2[index - 8u]; +} + +fn char_width(code: u32) -> f32 { + if code < 10u { + return CELL_WIDTH; + } + if code == COLON_CODE { + return COLON_WIDTH; + } + return DOT_WIDTH; +} + +// Bounds of one seven-segment segment, as (min_x, min_y, max_x, max_y). +fn segment_rect(segment: u32) -> vec4 { + let t = SEGMENT_THICKNESS; + let mid = DIGIT_HEIGHT * 0.5; + + switch segment { + case 0u: { return vec4(t, 0.0, CELL_WIDTH - t, t); } + case 1u: { return vec4(CELL_WIDTH - t, t, CELL_WIDTH, mid); } + case 2u: { return vec4(CELL_WIDTH - t, mid, CELL_WIDTH, DIGIT_HEIGHT - t); } + case 3u: { return vec4(t, DIGIT_HEIGHT - t, CELL_WIDTH - t, DIGIT_HEIGHT); } + case 4u: { return vec4(0.0, mid, t, DIGIT_HEIGHT - t); } + case 5u: { return vec4(0.0, t, t, mid); } + case 6u: { return vec4(t, mid - t * 0.5, CELL_WIDTH - t, mid + t * 0.5); } + default: { return vec4(0.0); } + } +} + +// Coverage of the rectangle [min_p, max_p] at point `p`. The interior +// is fully opaque. Alpha falls to zero over `feather` outside the edge, +// so touching rectangles union without seams. +fn rect_alpha(p: vec2, min_p: vec2, max_p: vec2, feather: f32) -> f32 { + let center = (min_p + max_p) * 0.5; + let half_size = (max_p - min_p) * 0.5; + let d = abs(p - center) - half_size; + let outside = length(max(d, vec2(0.0))); + let inside = min(max(d.x, d.y), 0.0); + return 1.0 - smoothstep(0.0, feather, outside + inside); +} + +fn circle_alpha(p: vec2, center: vec2, radius: f32, feather: f32) -> f32 { + return 1.0 - smoothstep(0.0, feather, length(p - center) - radius); +} + +// Coverage of one seven-segment digit with its origin at `origin`. +fn digit_alpha(p: vec2, origin: vec2, digit: u32, feather: f32) -> f32 { + if digit > 9u { + return 0.0; + } + + let local = p - origin; + let mask = DIGIT_MASKS[digit]; + var alpha = 0.0; + + for (var segment = 0u; segment < 7u; segment = segment + 1u) { + if (mask & (1u << segment)) != 0u { + let r = segment_rect(segment); + alpha = max(alpha, rect_alpha(local, r.xy, r.zw, feather)); + } + } + + return alpha; +} + +// Coverage of a colon or dot separator with its origin at `origin`. +fn separator_alpha(p: vec2, origin: vec2, code: u32, feather: f32) -> f32 { + let local = p - origin; + let center_x = char_width(code) * 0.5; + + if code == COLON_CODE { + let r = 0.095; + let top = circle_alpha(local, vec2(center_x, DIGIT_HEIGHT * 0.38), r, feather); + let bottom = circle_alpha(local, vec2(center_x, DIGIT_HEIGHT * 0.62), r, feather); + return max(top, bottom); + } + + if code == DOT_CODE { + return circle_alpha(local, vec2(center_x, DIGIT_HEIGHT - 0.095), 0.08, feather); + } + + return 0.0; +} + +// Coverage of the twelve clock characters. +fn chars_alpha(p: vec2, feather: f32) -> f32 { + // Skip the character loop outside the digit row. + if p.x < -feather || p.x > TOTAL_WIDTH + feather + || p.y < -feather || p.y > DIGIT_HEIGHT + feather { + return 0.0; + } + + var cursor = 0.0; + var alpha = 0.0; + + for (var index = 0u; index < CHAR_COUNT; index = index + 1u) { + let code = char_at(index); + let origin = vec2(cursor, 0.0); + + if code < 10u { + alpha = max(alpha, digit_alpha(p, origin, code, feather)); + } else { + alpha = max(alpha, separator_alpha(p, origin, code, feather)); + } + + cursor = cursor + char_width(code) + GAP; + } + + return alpha; +} + +// Coverage of the millisecond grid: filled cells in x, unfilled cells +// in y. Each row fills to show one millisecond digit. +fn grid_alpha(p: vec2, feather: f32) -> vec2 { + let grid_origin = + vec2((TOTAL_WIDTH - GRID_WIDTH) * 0.5, DIGIT_HEIGHT + GRID_TOP_GAP); + + // Skip the cell loop outside the grid. + let local = p - grid_origin; + if local.x < -feather || local.x > GRID_WIDTH + feather + || local.y < -feather || local.y > GRID_HEIGHT + feather { + return vec2(0.0, 0.0); + } + + var filled = 0.0; + var unfilled = 0.0; + + for (var row = 0u; row < GRID_ROWS; row = row + 1u) { + let row_digit = char_at(9u + row); + for (var column = 0u; column < GRID_COLUMNS; column = column + 1u) { + let cell_origin = grid_origin + vec2( + f32(column) * (GRID_CELL + GRID_COLUMN_GAP), + f32(row) * (GRID_CELL + GRID_ROW_GAP), + ); + let cell_alpha = rect_alpha(p, cell_origin, cell_origin + vec2(GRID_CELL), feather); + if column < row_digit { + filled = max(filled, cell_alpha); + } else { + unfilled = max(unfilled, cell_alpha); + } + } + } + + return vec2(filled, unfilled); +} + +@fragment +fn fs_main(in: VertexOut) -> @location(0) vec4 { + // Work in a space that is `aspect` wide and 1.0 tall. + let height = max(clock.viewport_size.y, 1.0); + let aspect = max(clock.viewport_size.x / height, 0.1); + let p = vec2(in.uv.x * aspect, in.uv.y); + + // Fit the clock into the frame with a margin, and center it. + let scale = min((aspect * 0.94) / TOTAL_WIDTH, 0.82 / GROUP_HEIGHT); + let scaled_size = vec2(TOTAL_WIDTH, GROUP_HEIGHT) * scale; + let origin = vec2((aspect - scaled_size.x) * 0.5, (1.0 - scaled_size.y) * 0.5); + let local_p = (p - origin) / scale; + + // About 1.5 output pixels of edge feather, in layout units. + let feather = 1.5 / (height * scale); + + let chars = chars_alpha(local_p, feather); + let grid = grid_alpha(local_p, feather); + let alpha = max(chars, grid.x); + + let color = max(EMPTY_CELL * grid.y, FOREGROUND * alpha); + return vec4(color, 1.0); +} diff --git a/livekit-capture/shaders/gradient.wgsl b/livekit-capture/shaders/gradient.wgsl new file mode 100644 index 000000000..7e5f6c07a --- /dev/null +++ b/livekit-capture/shaders/gradient.wgsl @@ -0,0 +1,5 @@ +// Animated color gradient: the built-in gradient pattern. +fn shade(uv: vec2) -> vec4 { + let color = 0.5 + 0.5 * cos(lk.time_s + uv.xyx * 4.0 + vec3(0.0, 2.0, 4.0)); + return vec4(color, 1.0); +} diff --git a/livekit-capture/shaders/logo.wgsl b/livekit-capture/shaders/logo.wgsl new file mode 100644 index 000000000..1d6f40073 --- /dev/null +++ b/livekit-capture/shaders/logo.wgsl @@ -0,0 +1,101 @@ +// Bouncing LiveKit logo: the built-in logo pattern. +// +// A white 7x7-cell tile carries the LiveKit glyph in black at its +// center. The tile moves in a straight line and reflects off the frame +// edges. Position is a pure function of time. +// +// All rectangles get about two pixels of edge feather. The soft edges +// make sub-pixel motion smooth, and they survive chroma subsampling and +// video encoding. + +// The glyph as a 5x5 bitmap. Each row is a 5-bit mask. The highest bit +// is the leftmost column: +// +// 1 0 0 0 1 +// 1 0 0 1 0 +// 1 0 1 0 0 +// 1 0 0 1 0 +// 1 1 1 0 1 +const GLYPH_ROWS: array = array(0x11u, 0x12u, 0x14u, 0x12u, 0x1du); + +// Cells per tile side, and the glyph offset into the tile, in cells. +const TILE_CELLS: f32 = 7.0; +const GLYPH_OFFSET: f32 = 1.0; + +// Tile height as a fraction of the frame height. This keeps the logo +// the same visual size at every resolution. +const LOGO_SIZE: f32 = 0.25; + +// Speed along each axis, in frame heights per second. The two values +// have no small common multiple, so the bounce path repeats slowly. +const SPEED: vec2 = vec2(0.23, 0.17); + +// Starting phase, so the logo does not start in a corner. +const START_PHASE: vec2 = vec2(0.34, 0.71); + +// White and black keep every edge luma-only. Luma has full resolution +// in 4:2:0 video, so these edges encode cleanly. +const TILE_COLOR: vec3 = vec3(1.0, 1.0, 1.0); +const GLYPH_COLOR: vec3 = vec3(0.0, 0.0, 0.0); +const BACKGROUND: vec3 = vec3(0.0, 0.0, 0.0); + +// Folds a growing phase into ping-pong motion between 0.0 and 1.0. +fn ping_pong(phase: f32) -> f32 { + return 1.0 - abs(1.0 - 2.0 * fract(phase * 0.5)); +} + +// Coverage of the rectangle [min_p, max_p] at point `p`. The interior +// is fully opaque. Alpha falls to zero over `feather` outside the edge, +// so touching rectangles union without seams. +fn rect_alpha(p: vec2, min_p: vec2, max_p: vec2, feather: f32) -> f32 { + let center = (min_p + max_p) * 0.5; + let half_size = (max_p - min_p) * 0.5; + let d = abs(p - center) - half_size; + let outside = length(max(d, vec2(0.0))); + let inside = min(max(d.x, d.y), 0.0); + return 1.0 - smoothstep(0.0, feather, outside + inside); +} + +fn shade(uv: vec2) -> vec4 { + // Work in a space that is `aspect` wide and 1.0 tall, so the cells + // stay square. Pixels in this space are 1.0 / height on both axes. + let height = max(lk.resolution.y, 1.0); + let aspect = lk.resolution.x / height; + let p = vec2(uv.x * aspect, uv.y); + + // Distance the logo can travel along each axis. The lower bound + // keeps the math finite when the frame is narrower than the logo. + let travel = max(vec2(aspect, 1.0) - vec2(LOGO_SIZE), vec2(0.0001)); + let phase = START_PHASE + lk.time_s * SPEED / travel; + let origin = vec2(ping_pong(phase.x), ping_pong(phase.y)) * travel; + + // About two output pixels of edge feather. + let feather = 2.0 / height; + + // Skip the coverage math outside the tile and its feather band. + let local = p - origin; + if local.x < -feather || local.x > LOGO_SIZE + feather + || local.y < -feather || local.y > LOGO_SIZE + feather { + return vec4(BACKGROUND, 1.0); + } + + // Coverage of the tile, and of the glyph cells inside it. + let tile = rect_alpha(p, origin, origin + vec2(LOGO_SIZE), feather); + + let cell_size = LOGO_SIZE / TILE_CELLS; + var glyph = 0.0; + for (var row = 0u; row < 5u; row = row + 1u) { + let bits = GLYPH_ROWS[row]; + for (var col = 0u; col < 5u; col = col + 1u) { + if ((bits >> (4u - col)) & 1u) == 0u { + continue; + } + let cell_min = origin + + (vec2(f32(col), f32(row)) + vec2(GLYPH_OFFSET)) * cell_size; + glyph = max(glyph, rect_alpha(p, cell_min, cell_min + vec2(cell_size), feather)); + } + } + + let logo = mix(TILE_COLOR, GLYPH_COLOR, glyph); + return vec4(mix(BACKGROUND, logo, tile), 1.0); +} diff --git a/livekit-capture/shaders/prelude.wgsl b/livekit-capture/shaders/prelude.wgsl new file mode 100644 index 000000000..2f74da385 --- /dev/null +++ b/livekit-capture/shaders/prelude.wgsl @@ -0,0 +1,34 @@ +// Prelude prepended to every pattern fragment snippet by the pattern +// video source. It declares the uniforms, draws one triangle that covers +// the full target, and calls the snippet's `shade` function once per +// pixel. +// +// Each pattern snippet must define `fn shade(uv: vec2) -> vec4`, +// and must not redeclare the names below. + +struct LkUniforms { + resolution: vec2, + time_s: f32, + frame_index: u32, +} + +@group(0) @binding(0) var lk: LkUniforms; + +struct LkVertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> LkVertexOutput { + let corner = vec2(f32((vertex_index << 1u) & 2u), f32(vertex_index & 2u)); + var out: LkVertexOutput; + out.position = vec4(corner * 2.0 - 1.0, 0.0, 1.0); + out.uv = vec2(corner.x, 1.0 - corner.y); + return out; +} + +@fragment +fn fs_main(in: LkVertexOutput) -> @location(0) vec4 { + return shade(in.uv); +} diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs new file mode 100644 index 000000000..2790a0db4 --- /dev/null +++ b/livekit-capture/src/encoded/h26x.rs @@ -0,0 +1,1109 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! H.264/H.265 parsing helpers: NAL-unit splitting, access-unit assembly, +//! and keyframe detection. + +use crate::{ + encoded::{EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit}, + primitive::VideoResolution, +}; +use bytes::Bytes; +use std::ops::Range; +use thiserror::Error; + +/// Error returned by the H26x parsing helpers. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum H26xParseError { + /// Encoded payload is empty. + #[error("encoded payload is empty")] + EmptyPayload, + /// H.265 NAL unit is too short to contain its header. + #[error("H.265 NAL unit is too short")] + H265NalTooShort, + /// Codec is not supported by the H26x parsing helpers. + #[error("H26x parsing does not support {0:?}")] + UnsupportedCodec(EncodedVideoCodec), + /// Encoded payload or transport data is malformed. + #[error("invalid encoded data: {0}")] + InvalidEncodedData(&'static str), +} + +/// Start code prepended to each NAL unit when assembling Annex-B payloads. +const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; + +/// Upper bound on bytes buffered while waiting for an access-unit boundary. +const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; + +/// Byte-stream access-unit parser shared by the encoded ingest sources. +/// +/// `push` appends bytes and returns at most one completed access unit; call +/// `drain` repeatedly to pull further access units already buffered, and +/// `flush` once at end of stream to emit the final pending access unit. +#[cfg(test)] +pub(crate) trait AccessUnitParser { + /// Appends bytes and returns the next complete access unit, if any. + fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError>; + + /// Returns the next complete access unit from already-buffered bytes. + fn drain(&mut self) -> Result, H26xParseError> { + self.push(&[]) + } + + /// Flushes remaining buffered bytes as the final access unit. + fn flush(&mut self) -> Result, H26xParseError>; +} + +/// Incremental parser that splits an H.264 or H.265 Annex-B byte stream +/// into access units. +/// +/// Call [`AnnexBAccessUnitParser::push`] as bytes arrive, and +/// [`AnnexBAccessUnitParser::flush`] at the end of the stream. +#[derive(Debug, Clone)] +pub struct AnnexBAccessUnitParser { + codec: EncodedVideoCodec, + pending: Vec, + // NAL ranges found in `pending`; the last range's end is provisional + // until the next start code (or flush) confirms it. + nal_ranges: Vec>, + // Offset up to which `pending` has been scanned for start codes. + scan_cursor: usize, + next_timestamp_us: i64, + frame_interval_us: i64, + resolution: VideoResolution, +} + +/// H.264/AVC length-prefixed parser state. +#[cfg(test)] +#[derive(Debug, Clone)] +pub(crate) struct AvcAccessUnitParser { + pending: Vec, + /// Complete NAL ranges found in `pending`. + nal_ranges: Vec>, + /// Offset of the first unparsed length prefix or incomplete NAL in `pending`. + scan_cursor: usize, + nal_length_size: u8, + next_timestamp_us: i64, + frame_interval_us: i64, + resolution: VideoResolution, +} + +impl AnnexBAccessUnitParser { + /// Creates a parser for an H.264 or H.265 Annex-B byte stream. + /// + /// Access units get timestamps that start at `start_timestamp_us` and + /// step by `frame_interval_us`. + pub fn new( + codec: EncodedVideoCodec, + start_timestamp_us: i64, + frame_interval_us: i64, + resolution: VideoResolution, + ) -> Result { + match codec { + EncodedVideoCodec::H264 | EncodedVideoCodec::H265 => {} + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + return Err(H26xParseError::UnsupportedCodec(codec)); + } + } + + Ok(Self { + codec, + pending: Vec::new(), + nal_ranges: Vec::new(), + scan_cursor: 0, + next_timestamp_us: start_timestamp_us, + frame_interval_us, + resolution, + }) + } + + /// Pushes encoded bytes and returns the next complete access unit, if + /// one is found. + /// + /// One call returns at most one access unit. Push an empty slice to + /// pull further access units that are already buffered. + pub fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError> { + self.pending.extend_from_slice(bytes); + self.drain_next(false) + } + + /// Flushes the remaining buffered bytes as the final access unit. + pub fn flush(&mut self) -> Result, H26xParseError> { + self.drain_next(true) + } + + fn drain_next( + &mut self, + at_eof: bool, + ) -> Result, H26xParseError> { + self.scan_pending(); + + if let Some(split_at) = + access_unit_split_index(self.codec, &self.pending, &self.nal_ranges)? + { + return self.take_access_unit(split_at); + } + if at_eof && self.nal_ranges.iter().any(|range| range.start < range.end) { + return self.take_access_unit(self.pending.len()); + } + if !at_eof && self.pending.len() > MAX_PENDING_ACCESS_UNIT_BYTES { + return Err(H26xParseError::InvalidEncodedData( + "access unit exceeds maximum buffered size", + )); + } + Ok(None) + } + + /// Scans bytes appended since the previous call, extending the cached NAL ranges. + fn scan_pending(&mut self) { + // Resume behind the previous scan end so a start code straddling the + // boundary is found, but never before the last NAL start so an + // already-found start code is not rediscovered. + let mut cursor = self.scan_cursor.saturating_sub(3); + if let Some(last) = self.nal_ranges.last() { + cursor = cursor.max(last.start); + } + while let Some((offset, prefix_len)) = find_start_code(&self.pending[cursor..]) { + let prefix_start = cursor + offset; + let nal_start = prefix_start + prefix_len; + if let Some(last) = self.nal_ranges.last_mut() { + last.end = prefix_start; + if last.start >= prefix_start { + self.nal_ranges.pop(); + } + } + self.nal_ranges.push(nal_start..nal_start); + cursor = nal_start; + } + if let Some(last) = self.nal_ranges.last_mut() { + last.end = self.pending.len(); + } + self.scan_cursor = self.pending.len(); + } + + fn take_access_unit( + &mut self, + byte_len: usize, + ) -> Result, H26xParseError> { + if byte_len == 0 { + return Ok(None); + } + + let access_unit = self.pending[..byte_len].to_vec(); + self.pending.drain(..byte_len); + self.nal_ranges.retain_mut(|range| { + if range.end <= byte_len { + return false; + } + range.start -= byte_len; + range.end -= byte_len; + true + }); + self.scan_cursor -= byte_len; + let timestamp_us = self.next_timestamp_us; + self.next_timestamp_us = self.next_timestamp_us.saturating_add(self.frame_interval_us); + access_unit_from_annex_b( + self.codec, + Bytes::from(access_unit), + timestamp_us, + self.resolution, + ) + .map(Some) + } +} + +#[cfg(test)] +impl AccessUnitParser for AnnexBAccessUnitParser { + fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError> { + AnnexBAccessUnitParser::push(self, bytes) + } + + fn flush(&mut self) -> Result, H26xParseError> { + AnnexBAccessUnitParser::flush(self) + } +} + +#[cfg(test)] +impl AvcAccessUnitParser { + /// Creates a parser for H.264/AVC length-prefixed byte streams. + pub(crate) fn new( + nal_length_size: u8, + start_timestamp_us: i64, + frame_interval_us: i64, + resolution: VideoResolution, + ) -> Result { + validate_avc_nal_length_size(nal_length_size)?; + + Ok(Self { + pending: Vec::new(), + nal_ranges: Vec::new(), + scan_cursor: 0, + nal_length_size, + next_timestamp_us: start_timestamp_us, + frame_interval_us, + resolution, + }) + } + + /// Pushes encoded bytes and returns the next complete access unit if one is found. + pub(crate) fn push( + &mut self, + bytes: &[u8], + ) -> Result, H26xParseError> { + self.pending.extend_from_slice(bytes); + self.drain_next(false) + } + + /// Flushes the pending bytes as the final access unit. + pub(crate) fn flush(&mut self) -> Result, H26xParseError> { + self.drain_next(true) + } + + fn drain_next( + &mut self, + at_eof: bool, + ) -> Result, H26xParseError> { + self.scan_pending(at_eof)?; + + if let Some(split_at) = avc_access_unit_split_index( + &self.pending, + &self.nal_ranges, + self.nal_length_size as usize, + )? { + return self.take_access_unit(split_at); + } + if at_eof && !self.nal_ranges.is_empty() { + return self.take_access_unit(self.pending.len()); + } + if !at_eof && self.pending.len() > MAX_PENDING_ACCESS_UNIT_BYTES { + return Err(H26xParseError::InvalidEncodedData( + "access unit exceeds maximum buffered size", + )); + } + Ok(None) + } + + /// Parses length-prefixed NAL units appended since the previous call. + fn scan_pending(&mut self, at_eof: bool) -> Result<(), H26xParseError> { + let nal_length_size = self.nal_length_size as usize; + while self.scan_cursor < self.pending.len() { + if self.pending.len() - self.scan_cursor < nal_length_size { + if at_eof { + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL length")); + } + break; + } + + let nal_start = self.scan_cursor + nal_length_size; + let nal_len = read_avc_nal_length(&self.pending[self.scan_cursor..nal_start]); + if nal_len == 0 { + return Err(H26xParseError::InvalidEncodedData("empty AVC NAL unit")); + } + + let Some(nal_end) = nal_start.checked_add(nal_len) else { + return Err(H26xParseError::InvalidEncodedData("AVC NAL unit length overflow")); + }; + if nal_end > self.pending.len() { + if at_eof { + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL unit")); + } + break; + } + + self.nal_ranges.push(nal_start..nal_end); + self.scan_cursor = nal_end; + } + Ok(()) + } + + fn take_access_unit( + &mut self, + byte_len: usize, + ) -> Result, H26xParseError> { + if byte_len == 0 { + return Ok(None); + } + + let access_unit = self.pending[..byte_len].to_vec(); + self.pending.drain(..byte_len); + self.nal_ranges.retain_mut(|range| { + if range.end <= byte_len { + return false; + } + range.start -= byte_len; + range.end -= byte_len; + true + }); + self.scan_cursor -= byte_len; + let timestamp_us = self.next_timestamp_us; + self.next_timestamp_us = self.next_timestamp_us.saturating_add(self.frame_interval_us); + access_unit_from_h264_avc(&access_unit, self.nal_length_size, timestamp_us, self.resolution) + .map(Some) + } +} + +#[cfg(test)] +impl AccessUnitParser for AvcAccessUnitParser { + fn push(&mut self, bytes: &[u8]) -> Result, H26xParseError> { + AvcAccessUnitParser::push(self, bytes) + } + + fn flush(&mut self) -> Result, H26xParseError> { + AvcAccessUnitParser::flush(self) + } +} + +/// Returns NAL-unit byte ranges for an Annex-B access unit or stream chunk. +pub fn annex_b_nal_ranges(bytes: &[u8]) -> Vec> { + let mut ranges = Vec::new(); + let mut cursor = 0; + let mut current_start = None; + + while let Some((prefix_start, prefix_len)) = find_start_code(&bytes[cursor..]) { + let prefix_start = cursor + prefix_start; + let nal_start = prefix_start + prefix_len; + if let Some(start) = current_start.replace(nal_start) { + if start < prefix_start { + ranges.push(start..prefix_start); + } + } + cursor = nal_start; + } + + if let Some(start) = current_start { + if start < bytes.len() { + ranges.push(start..bytes.len()); + } + } + + ranges +} + +/// Returns borrowed NAL units from an Annex-B buffer. +pub fn annex_b_nalus(bytes: &[u8]) -> Vec<&[u8]> { + annex_b_nal_ranges(bytes) + .into_iter() + .map(|range| &bytes[range]) + .filter(|nal| !nal.is_empty()) + .collect() +} + +/// Creates an Annex-B access unit from H.264/AVC length-prefixed NAL units. +pub fn access_unit_from_h264_avc( + payload: &[u8], + nal_length_size: u8, + timestamp_us: i64, + resolution: VideoResolution, +) -> Result { + let nals = avc_nalus(payload, nal_length_size)?; + access_unit_from_nalus(EncodedVideoCodec::H264, &nals, timestamp_us, resolution) +} + +/// Creates an access unit from an Annex-B buffer. +pub fn access_unit_from_annex_b( + codec: EncodedVideoCodec, + payload: Bytes, + timestamp_us: i64, + resolution: VideoResolution, +) -> Result { + if payload.is_empty() { + return Err(H26xParseError::EmptyPayload); + } + + let frame_type = if is_keyframe_annex_b(codec, &payload)? { + EncodedFrameType::Key + } else { + EncodedFrameType::Delta + }; + Ok(OwnedEncodedAccessUnit::new(codec, payload, timestamp_us, frame_type, resolution)) +} + +/// Creates an Annex-B access unit from raw NAL units. +pub fn access_unit_from_nalus( + codec: EncodedVideoCodec, + nal_units: &[&[u8]], + timestamp_us: i64, + resolution: VideoResolution, +) -> Result { + let payload = Bytes::from(annex_b_payload(nal_units)?); + access_unit_from_annex_b(codec, payload, timestamp_us, resolution) +} + +/// Returns `true` when an Annex-B access unit is a key frame: an IDR +/// picture for H.264, or parameter sets (VPS/SPS/PPS) plus an IDR picture +/// for H.265. +pub fn is_keyframe_annex_b(codec: EncodedVideoCodec, bytes: &[u8]) -> Result { + let nals = annex_b_nalus(bytes); + is_keyframe_nalus(codec, &nals) +} + +/// Returns true when the NAL units form a WebRTC-usable key frame. +fn is_keyframe_nalus( + codec: EncodedVideoCodec, + nal_units: &[&[u8]], +) -> Result { + match codec { + EncodedVideoCodec::H264 => { + nal_units.iter().try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == 5)) + } + EncodedVideoCodec::H265 => { + let mut has_vps = false; + let mut has_sps = false; + let mut has_pps = false; + let mut has_idr = false; + + for nal in nal_units { + match h265_nal_type(nal)? { + 32 => has_vps = true, + 33 => has_sps = true, + 34 => has_pps = true, + 19 | 20 => has_idr = true, + _ => {} + } + } + + Ok(has_vps && has_sps && has_pps && has_idr) + } + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + Err(H26xParseError::UnsupportedCodec(codec)) + } + } +} + +fn h264_nal_type(nal: &[u8]) -> Result { + let header = nal.first().ok_or(H26xParseError::EmptyPayload)?; + Ok(header & 0x1f) +} + +fn h265_nal_type(nal: &[u8]) -> Result { + if nal.is_empty() { + return Err(H26xParseError::EmptyPayload); + } + if nal.len() < 2 { + return Err(H26xParseError::H265NalTooShort); + } + Ok((nal[0] >> 1) & 0x3f) +} + +fn annex_b_payload(nal_units: &[&[u8]]) -> Result, H26xParseError> { + if nal_units.is_empty() { + return Err(H26xParseError::EmptyPayload); + } + let len = nal_units.iter().try_fold(0usize, |len, nal| { + if nal.is_empty() { + Err(H26xParseError::EmptyPayload) + } else { + Ok(len + ANNEX_B_START_CODE.len() + nal.len()) + } + })?; + + let mut payload = Vec::with_capacity(len); + for nal in nal_units { + payload.extend_from_slice(&ANNEX_B_START_CODE); + payload.extend_from_slice(nal); + } + Ok(payload) +} + +fn access_unit_split_index( + codec: EncodedVideoCodec, + bytes: &[u8], + ranges: &[Range], +) -> Result, H26xParseError> { + match access_unit_boundary_nal(codec, bytes, ranges)? { + Some(index) => split_start_code_index(bytes, ranges[index].start).map(Some), + None => Ok(None), + } +} + +#[cfg(test)] +fn avc_access_unit_split_index( + bytes: &[u8], + ranges: &[Range], + nal_length_size: usize, +) -> Result, H26xParseError> { + match access_unit_boundary_nal(EncodedVideoCodec::H264, bytes, ranges)? { + Some(index) => ranges[index] + .start + .checked_sub(nal_length_size) + .ok_or(H26xParseError::InvalidEncodedData("missing AVC NAL length")) + .map(Some), + None => Ok(None), + } +} + +/// Returns the index of the first NAL that starts a new access unit, once at +/// least one VCL NAL has been seen in the current one. +fn access_unit_boundary_nal( + codec: EncodedVideoCodec, + bytes: &[u8], + ranges: &[Range], +) -> Result, H26xParseError> { + let mut seen_vcl = false; + for (index, range) in ranges.iter().enumerate() { + let nal = &bytes[range.clone()]; + // The final NAL may still be streaming in; wait for its header. + if index + 1 == ranges.len() && nal.len() < min_nal_header_len(codec) { + return Ok(None); + } + if seen_vcl && starts_new_access_unit(codec, nal)? { + return Ok(Some(index)); + } + seen_vcl |= is_vcl_nal(codec, nal)?; + } + Ok(None) +} + +fn min_nal_header_len(codec: EncodedVideoCodec) -> usize { + match codec { + EncodedVideoCodec::H265 => 2, + _ => 1, + } +} + +fn starts_new_access_unit(codec: EncodedVideoCodec, nal: &[u8]) -> Result { + Ok(match codec { + EncodedVideoCodec::H264 => match h264_nal_type(nal)? { + // Prefix SEI(6), SPS(7), PPS(8), and AUD(9) open a new access unit. + 6..=9 => true, + // A VCL NAL opens a new picture when first_mb_in_slice == 0: + // ue(v) == 0 is a lone 1 bit, so the first RBSP bit after the + // header is set. The header byte is nonzero, so the next byte + // cannot be an emulation-prevention byte. + 1..=5 => nal.len() >= 2 && nal[1] & 0x80 != 0, + _ => false, + }, + EncodedVideoCodec::H265 => match h265_nal_type(nal)? { + // VPS(32), SPS(33), PPS(34), AUD(35), and prefix SEI(39). + 32..=35 | 39 => true, + // A VCL NAL opens a new picture when + // first_slice_segment_in_pic_flag (the bit after the 2-byte + // header) is set. nuh_temporal_id_plus1 makes the second header + // byte nonzero, so the next byte cannot be an + // emulation-prevention byte. + 0..=31 => nal.len() >= 3 && nal[2] & 0x80 != 0, + _ => false, + }, + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + return Err(H26xParseError::UnsupportedCodec(codec)); + } + }) +} + +fn split_start_code_index(bytes: &[u8], nal_start: usize) -> Result { + if nal_start >= 4 && bytes[nal_start - 4..nal_start] == [0, 0, 0, 1] { + return Ok(nal_start - 4); + } + if nal_start >= 3 && bytes[nal_start - 3..nal_start] == [0, 0, 1] { + return Ok(nal_start - 3); + } + Err(H26xParseError::InvalidEncodedData("missing Annex-B start code")) +} + +fn is_vcl_nal(codec: EncodedVideoCodec, nal: &[u8]) -> Result { + Ok(match codec { + EncodedVideoCodec::H264 => (1..=5).contains(&h264_nal_type(nal)?), + EncodedVideoCodec::H265 => h265_nal_type(nal)? <= 31, + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + return Err(H26xParseError::UnsupportedCodec(codec)); + } + }) +} + +fn find_start_code(bytes: &[u8]) -> Option<(usize, usize)> { + let mut idx = 0; + while idx + 3 <= bytes.len() { + if bytes[idx..].starts_with(&[0, 0, 1]) { + return Some((idx, 3)); + } + if idx + 4 <= bytes.len() && bytes[idx..].starts_with(&[0, 0, 0, 1]) { + return Some((idx, 4)); + } + idx += 1; + } + None +} + +fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, H26xParseError> { + let ranges = avc_nal_ranges(payload, nal_length_size, true)?; + if ranges.is_empty() { + return Err(H26xParseError::EmptyPayload); + } + Ok(ranges.into_iter().map(|range| &payload[range]).collect()) +} + +fn avc_nal_ranges( + bytes: &[u8], + nal_length_size: u8, + at_eof: bool, +) -> Result>, H26xParseError> { + validate_avc_nal_length_size(nal_length_size)?; + + let nal_length_size = nal_length_size as usize; + let mut ranges = Vec::new(); + let mut cursor = 0; + while cursor < bytes.len() { + if bytes.len() - cursor < nal_length_size { + if at_eof { + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL length")); + } + break; + } + + let nal_len = read_avc_nal_length(&bytes[cursor..cursor + nal_length_size]); + cursor += nal_length_size; + if nal_len == 0 { + return Err(H26xParseError::InvalidEncodedData("empty AVC NAL unit")); + } + + let Some(nal_end) = cursor.checked_add(nal_len) else { + return Err(H26xParseError::InvalidEncodedData("AVC NAL unit length overflow")); + }; + if nal_end > bytes.len() { + if at_eof { + return Err(H26xParseError::InvalidEncodedData("truncated AVC NAL unit")); + } + break; + } + + ranges.push(cursor..nal_end); + cursor = nal_end; + } + + Ok(ranges) +} + +fn read_avc_nal_length(bytes: &[u8]) -> usize { + bytes.iter().fold(0usize, |len, byte| (len << 8) | usize::from(*byte)) +} + +fn validate_avc_nal_length_size(nal_length_size: u8) -> Result<(), H26xParseError> { + if (1..=4).contains(&nal_length_size) { + return Ok(()); + } + Err(H26xParseError::InvalidEncodedData("invalid AVC NAL length size")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn h264_keyframe_requires_idr_nal() { + let sps = [0x67, 1, 2, 3]; + let idr = [0x65, 4, 5, 6]; + let non_idr = [0x61, 1, 2]; + + assert!(is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &idr]).unwrap()); + assert!(!is_keyframe_nalus(EncodedVideoCodec::H264, &[&sps, &non_idr]).unwrap()); + } + + #[test] + fn h265_keyframe_requires_parameter_sets_and_idr() { + let vps = [0x40, 1, 2]; + let sps = [0x42, 1, 2]; + let pps = [0x44, 1, 2]; + let idr_w_radl = [19 << 1, 1, 3]; + let cra = [21 << 1, 1, 3]; + + assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &idr_w_radl]).unwrap()); + assert!( + is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &idr_w_radl]).unwrap() + ); + assert!(!is_keyframe_nalus(EncodedVideoCodec::H265, &[&vps, &sps, &pps, &cra]).unwrap()); + } + + #[test] + fn h265_rejects_too_short_nal_header() { + let err = is_keyframe_nalus(EncodedVideoCodec::H265, &[&[0x26]]).unwrap_err(); + assert_eq!(err, H26xParseError::H265NalTooShort); + } + + #[test] + fn annex_b_payload_prefixes_each_nal_unit() { + let payload = annex_b_payload(&[&[0x67, 1, 2, 3], &[0x65, 4, 5, 6]]).unwrap(); + assert_eq!(payload, vec![0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5, 6]); + } + + #[test] + fn annex_b_payload_rejects_empty_input() { + assert_eq!(annex_b_payload(&[]).unwrap_err(), H26xParseError::EmptyPayload); + assert_eq!(annex_b_payload(&[&[]]).unwrap_err(), H26xParseError::EmptyPayload); + } + + #[test] + fn splits_annex_b_nals_with_three_and_four_byte_prefixes() { + let bytes = [0, 0, 1, 0x67, 1, 0, 0, 0, 1, 0x65, 2, 3]; + let nals = annex_b_nalus(&bytes); + assert_eq!(nals, vec![&[0x67, 1][..], &[0x65, 2, 3][..]]); + } + + #[test] + fn detects_h264_keyframe_from_annex_b() { + let bytes = [0, 0, 0, 1, 0x61, 1, 0, 0, 0, 1, 0x65, 2]; + assert!(is_keyframe_annex_b(EncodedVideoCodec::H264, &bytes).unwrap()); + } + + #[test] + fn access_unit_from_avc_converts_length_prefixed_nals() { + let bytes = [0, 0, 0, 4, 0x67, 1, 2, 3, 0, 0, 0, 3, 0x65, 4, 5]; + let au = access_unit_from_h264_avc(&bytes, 4, 10, VideoResolution::new(640, 480)).unwrap(); + + assert_eq!(au.codec, EncodedVideoCodec::H264); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x67, 1, 2, 3, 0, 0, 0, 1, 0x65, 4, 5]); + } + + #[test] + fn access_unit_from_avc_supports_two_byte_lengths() { + let bytes = [0, 2, 0x61, 1]; + let au = access_unit_from_h264_avc(&bytes, 2, 10, VideoResolution::new(640, 480)).unwrap(); + + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x61, 1]); + } + + #[test] + fn access_unit_from_avc_rejects_truncated_nal() { + let err = + access_unit_from_h264_avc(&[0, 0, 0, 3, 0x65], 4, 10, VideoResolution::new(640, 480)) + .unwrap_err(); + + assert_eq!(err, H26xParseError::InvalidEncodedData("truncated AVC NAL unit")); + } + + #[test] + fn parser_flushes_final_access_unit() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 100, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + assert!(parser.push(&[0, 0, 1, 0x65, 1, 2]).unwrap().is_none()); + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 100); + assert_eq!(au.frame_type, EncodedFrameType::Key); + } + + #[test] + fn parser_splits_at_next_access_unit_delimiter() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 100, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + let stream = + [0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x65, 1, 2, 0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x41, 3]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 100); + assert_eq!(au.payload.as_ref(), &[0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x65, 1, 2]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_433); + assert_eq!(au.payload.as_ref(), &[0, 0, 1, 0x09, 0x10, 0, 0, 1, 0x41, 3]); + } + + #[test] + fn avc_parser_splits_at_next_access_unit_delimiter() { + let mut parser = + AvcAccessUnitParser::new(4, 100, 33_333, VideoResolution::new(640, 480)).unwrap(); + let stream = [ + 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 3, 0x65, 1, 2, 0, 0, 0, 2, 0x09, 0x10, 0, 0, 0, 2, + 0x41, 3, + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 100); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x09, 0x10, 0, 0, 0, 1, 0x65, 1, 2]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_433); + assert_eq!(au.payload.as_ref(), &[0, 0, 0, 1, 0x09, 0x10, 0, 0, 0, 1, 0x41, 3]); + } + + #[test] + fn splits_aud_less_h264_stream_per_frame() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + let stream = [ + 0, 0, 0, 1, 0x67, 0x42, 0x00, 0x1e, // SPS + 0, 0, 0, 1, 0x68, 0xce, // PPS + 0, 0, 1, 0x65, 0x88, 0x84, 0x21, // IDR slice, first_mb_in_slice == 0 + 0, 0, 1, 0x41, 0x9a, 0x22, // P slice, first_mb_in_slice == 0 + 0, 0, 1, 0x41, 0x9a, 0x33, // P slice, first_mb_in_slice == 0 + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..21]); + + let au = parser.drain().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &stream[21..27]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 66_666); + assert_eq!(au.payload.as_ref(), &stream[27..]); + } + + #[test] + fn keeps_multi_slice_h264_access_unit_together() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + let stream = [ + 0, 0, 1, 0x65, 0x88, 0x11, // IDR slice, first_mb_in_slice == 0 + 0, 0, 1, 0x65, 0x21, 0x22, // IDR slice, first_mb_in_slice != 0 + 0, 0, 1, 0x41, 0x9a, 0x33, // next picture + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..12]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.payload.as_ref(), &stream[12..]); + } + + #[test] + fn splits_aud_less_h265_stream_per_frame() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H265, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + let stream = [ + 0, 0, 0, 1, 0x40, 0x01, 0x0c, // VPS + 0, 0, 0, 1, 0x42, 0x01, 0x02, // SPS + 0, 0, 0, 1, 0x44, 0x01, 0x03, // PPS + 0, 0, 1, 0x26, 0x01, 0xaf, + 0x04, // IDR_W_RADL, first_slice_segment_in_pic_flag == 1 + 0, 0, 1, 0x02, 0x01, 0xd0, 0x05, // TRAIL_R, first_slice_segment_in_pic_flag == 1 + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..28]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &stream[28..]); + } + + #[test] + fn keeps_multi_slice_h265_access_unit_together() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H265, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + let stream = [ + 0, 0, 1, 0x26, 0x01, 0xaf, + 0x11, // IDR slice, first_slice_segment_in_pic_flag == 1 + 0, 0, 1, 0x26, 0x01, 0x40, + 0x22, // IDR slice, first_slice_segment_in_pic_flag == 0 + 0, 0, 1, 0x02, 0x01, 0xd0, 0x33, // next picture + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Delta); + assert_eq!(au.payload.as_ref(), &stream[..14]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.payload.as_ref(), &stream[14..]); + } + + #[test] + fn groups_parameter_sets_with_following_frame() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + let stream = [ + 0, 0, 1, 0x67, 0x42, 0x1e, // SPS + 0, 0, 1, 0x68, 0xce, // PPS + 0, 0, 1, 0x65, 0x88, 0x11, // IDR + 0, 0, 1, 0x67, 0x42, 0x1e, // SPS + 0, 0, 1, 0x68, 0xce, // PPS + 0, 0, 1, 0x65, 0x88, 0x22, // IDR + ]; + + let au = parser.push(&stream).unwrap().unwrap(); + assert_eq!(au.timestamp_us, 0); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[..17]); + + let au = parser.flush().unwrap().unwrap(); + assert_eq!(au.timestamp_us, 33_333); + assert_eq!(au.frame_type, EncodedFrameType::Key); + assert_eq!(au.payload.as_ref(), &stream[17..]); + } + + fn collect_units( + parser: &mut impl AccessUnitParser, + stream: &[u8], + chunk_size: usize, + ) -> Vec<(Vec, i64, EncodedFrameType)> { + let mut units = Vec::new(); + for chunk in stream.chunks(chunk_size) { + let mut unit = parser.push(chunk).unwrap(); + while let Some(au) = unit { + units.push((au.payload.to_vec(), au.timestamp_us, au.frame_type)); + unit = parser.drain().unwrap(); + } + } + let mut unit = parser.flush().unwrap(); + while let Some(au) = unit { + units.push((au.payload.to_vec(), au.timestamp_us, au.frame_type)); + unit = parser.flush().unwrap(); + } + units + } + + fn assert_chunked_matches_one_shot( + make_parser: impl Fn() -> P, + stream: &[u8], + expected_units: usize, + ) { + let baseline = collect_units(&mut make_parser(), stream, stream.len()); + assert_eq!(baseline.len(), expected_units); + for chunk_size in [1, 7] { + assert_eq!(collect_units(&mut make_parser(), stream, chunk_size), baseline); + } + } + + #[test] + fn chunked_pushes_match_one_shot_parsing() { + let h264_annex_b = [ + 0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1e, // SPS + 0, 0, 0, 1, 0x68, 0xce, 0x3c, 0x80, // PPS + 0, 0, 1, 0x65, 0x88, 0x84, 0x00, 0x01, // IDR, first_mb_in_slice == 0 + 0, 0, 1, 0x41, 0x9a, 0x02, // P, first_mb_in_slice == 0 + 0, 0, 1, 0x09, 0x10, // AUD + 0, 0, 1, 0x41, 0x9a, 0x03, // P + 0, 0, 0, 1, 0x41, 0x9a, 0x04, 0x00, // P, first_mb_in_slice == 0 + ]; + assert_chunked_matches_one_shot( + || { + AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap() + }, + &h264_annex_b, + 4, + ); + + let h265_annex_b = [ + 0, 0, 0, 1, 0x40, 0x01, 0x0c, // VPS + 0, 0, 0, 1, 0x42, 0x01, 0x02, // SPS + 0, 0, 0, 1, 0x44, 0x01, 0x03, // PPS + 0, 0, 1, 0x26, 0x01, 0xaf, 0x08, // IDR_W_RADL + 0, 0, 1, 0x02, 0x01, 0xd0, 0x09, // TRAIL_R + 0, 0, 1, 0x46, 0x01, 0x50, // AUD + 0, 0, 1, 0x02, 0x01, 0xd0, 0x0a, // TRAIL_R + ]; + assert_chunked_matches_one_shot( + || { + AnnexBAccessUnitParser::new( + EncodedVideoCodec::H265, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap() + }, + &h265_annex_b, + 3, + ); + + let h264_avc = [ + 0, 0, 0, 4, 0x67, 0x64, 0x00, 0x1e, // SPS + 0, 0, 0, 2, 0x68, 0xce, // PPS + 0, 0, 0, 4, 0x65, 0x88, 0x84, 0x00, // IDR, first_mb_in_slice == 0 + 0, 0, 0, 3, 0x41, 0x9a, 0x02, // P, first_mb_in_slice == 0 + 0, 0, 0, 2, 0x09, 0x10, // AUD + 0, 0, 0, 3, 0x41, 0x9a, 0x03, // P + ]; + assert_chunked_matches_one_shot( + || AvcAccessUnitParser::new(4, 0, 33_333, VideoResolution::new(640, 480)).unwrap(), + &h264_avc, + 3, + ); + } + + #[test] + fn rejects_pending_access_unit_over_size_cap() { + let mut parser = AnnexBAccessUnitParser::new( + EncodedVideoCodec::H264, + 0, + 33_333, + VideoResolution::new(640, 480), + ) + .unwrap(); + assert!(parser.push(&[0, 0, 1, 0x65, 0x88]).unwrap().is_none()); + + let err = parser.push(&vec![0xff; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); + assert_eq!( + err, + H26xParseError::InvalidEncodedData("access unit exceeds maximum buffered size") + ); + } + + #[test] + fn avc_rejects_pending_access_unit_over_size_cap() { + let mut parser = + AvcAccessUnitParser::new(4, 0, 33_333, VideoResolution::new(640, 480)).unwrap(); + let nal_len = (MAX_PENDING_ACCESS_UNIT_BYTES + 1) as u32; + assert!(parser.push(&nal_len.to_be_bytes()).unwrap().is_none()); + + let err = parser.push(&vec![0x41; MAX_PENDING_ACCESS_UNIT_BYTES]).unwrap_err(); + assert_eq!( + err, + H26xParseError::InvalidEncodedData("access unit exceeds maximum buffered size") + ); + } +} diff --git a/livekit-capture/src/encoded/mod.rs b/livekit-capture/src/encoded/mod.rs new file mode 100644 index 000000000..5d696a77f --- /dev/null +++ b/livekit-capture/src/encoded/mod.rs @@ -0,0 +1,198 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pre-encoded video: the source trait, access units, and the pump. +//! +//! A source produces [`OwnedEncodedAccessUnit`]s, and [`EncodedVideoPump`] +//! publishes them to an RTC track as passthrough — without re-encoding. +//! +//! [`EncodedVideoSource`] is object-safe and implemented for `Box`, +//! so sources constructed dynamically run through the same generic pump. + +use crate::{error::SourceError, primitive::VideoResolution, pump::PumpStop}; +use bytes::Bytes; +use livekit::{ + options::VideoCodec, + webrtc::{ + video_frame::{ + EncodedFrameType as RtcEncodedFrameType, EncodedVideoCodec as RtcEncodedVideoCodec, + }, + video_source::EncodedRateControl, + }, +}; + +pub mod h26x; +mod pump; +pub use pump::EncodedVideoPump; + +/// Source of pre-encoded video access units, such as an encoding pipeline. +pub trait EncodedVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Codec produced by this source. The codec is fixed for the source's + /// lifetime. + fn codec(&self) -> EncodedVideoCodec; + + /// Blocks until the next access unit is available. Returns `Ok(None)` + /// at the end of the stream. + /// + /// Implementations must return `Ok(None)` promptly once `stop` fires: + /// integrate the token into the blocking wait, or bound each wait to + /// about one frame interval. The pump uses the token to tell a stop + /// from the end of the stream. + /// + /// Access units must carry a non-empty payload. The pump reports an + /// empty payload as a source error. + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError>; + + /// Forwards a downstream keyframe request (PLI/FIR, late subscriber) to + /// the producer so it can emit a keyframe. + /// + /// The default implementation does nothing, for sources that cannot + /// influence the upstream encoder. + fn request_keyframe(&mut self) {} + + /// Forwards a downstream rate-control target to the producer. + /// + /// The default implementation does nothing, for sources that cannot + /// influence the upstream encoder. + fn update_rate_control(&mut self, _target: EncodedRateControl) {} +} + +/// Encoded video codec carried by an [`OwnedEncodedAccessUnit`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "lowercase") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum EncodedVideoCodec { + /// H.264/AVC video. + H264, + /// H.265/HEVC video. + H265, + /// VP8 video. + VP8, + /// VP9 video. + VP9, + /// AV1 video. + AV1, +} + +/// Encoded video frame type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncodedFrameType { + /// A key frame. + Key, + /// A delta frame. + Delta, +} + +/// Owned encoded video access unit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OwnedEncodedAccessUnit { + /// Codec of the payload. + pub codec: EncodedVideoCodec, + /// Encoded payload bytes. + pub payload: Bytes, + /// Capture timestamp in microseconds. + pub timestamp_us: i64, + /// Frame type. + pub frame_type: EncodedFrameType, + /// Frame resolution in pixels. + pub resolution: VideoResolution, +} + +impl OwnedEncodedAccessUnit { + /// Creates an access unit. + pub fn new( + codec: EncodedVideoCodec, + payload: impl Into, + timestamp_us: i64, + frame_type: EncodedFrameType, + resolution: VideoResolution, + ) -> Self { + Self { codec, payload: payload.into(), timestamp_us, frame_type, resolution } + } +} + +impl From for VideoCodec { + fn from(value: EncodedVideoCodec) -> Self { + match value { + EncodedVideoCodec::H264 => Self::H264, + EncodedVideoCodec::H265 => Self::H265, + EncodedVideoCodec::VP8 => Self::VP8, + EncodedVideoCodec::VP9 => Self::VP9, + EncodedVideoCodec::AV1 => Self::AV1, + } + } +} + +impl From for RtcEncodedVideoCodec { + fn from(value: EncodedVideoCodec) -> Self { + match value { + EncodedVideoCodec::H264 => Self::H264, + EncodedVideoCodec::H265 => Self::H265, + EncodedVideoCodec::VP8 => Self::VP8, + EncodedVideoCodec::VP9 => Self::VP9, + EncodedVideoCodec::AV1 => Self::AV1, + } + } +} + +impl From for RtcEncodedFrameType { + fn from(value: EncodedFrameType) -> Self { + match value { + EncodedFrameType::Key => Self::Key, + EncodedFrameType::Delta => Self::Delta, + } + } +} + +impl EncodedVideoSource for Box { + fn resolution(&self) -> VideoResolution { + (**self).resolution() + } + + fn codec(&self) -> EncodedVideoCodec { + (**self).codec() + } + + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError> { + (**self).next_access_unit(stop) + } + + fn request_keyframe(&mut self) { + (**self).request_keyframe() + } + + fn update_rate_control(&mut self, target: EncodedRateControl) { + (**self).update_rate_control(target) + } +} + +// Object safety is part of this trait's contract: dynamic applications box +// sources at their edge and drive them through the same generic pumps. +const _: () = { + fn _assert_object_safe(_: &dyn EncodedVideoSource) {} +}; diff --git a/livekit-capture/src/encoded/pump.rs b/livekit-capture/src/encoded/pump.rs new file mode 100644 index 000000000..a7617b070 --- /dev/null +++ b/livekit-capture/src/encoded/pump.rs @@ -0,0 +1,293 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pumps encoded access units from a capture source into an RTC video +//! source. + +use crate::{ + encoded::{EncodedFrameType, EncodedVideoSource, OwnedEncodedAccessUnit}, + error::SourceError, + pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, +}; +use livekit::{ + options::{TrackPublishOptions, VideoEncoderBackend}, + webrtc::{ + video_frame::{EncodedVideoFrame, FrameMetadata}, + video_source::{native::NativeVideoSource, RtcVideoSource}, + }, +}; +use std::{fmt, io}; + +/// Callback that supplies packet-trailer metadata for an access unit. +type FrameMetadataFn = Box Option + Send>; + +/// Pumps an [`EncodedVideoSource`] into an RTC video source as passthrough, +/// without re-encoding. +/// +/// Downstream keyframe requests and rate-control targets are forwarded to +/// the source between access units. Delta frames that arrive before the +/// first keyframe are dropped, because decoding can only start at a +/// keyframe. +pub struct EncodedVideoPump { + source: S, + rtc_source: NativeVideoSource, + stop: PumpStop, + frame_metadata: Option, +} + +impl EncodedVideoPump { + /// Creates a pump for an encoded source and builds the matching RTC + /// source. + pub fn new(source: S) -> Self { + let rtc_source = NativeVideoSource::new_encoded(source.resolution().into()); + Self { source, rtc_source, stop: PumpStop::new(), frame_metadata: None } + } + + /// Sets a callback that supplies packet-trailer metadata for each access + /// unit. + /// + /// Subscribers receive metadata only when the matching + /// [`TrackPublishOptions::frame_metadata_features`] are enabled on the + /// published track. + pub fn with_frame_metadata( + mut self, + frame_metadata: impl FnMut(&OwnedEncodedAccessUnit) -> Option + Send + 'static, + ) -> Self { + self.frame_metadata = Some(Box::new(frame_metadata)); + self + } + + /// Returns the RTC source to create the local track with. + pub fn rtc_source(&self) -> RtcVideoSource { + RtcVideoSource::Native(self.rtc_source.clone()) + } + + /// Returns publish options for encoded passthrough. + pub fn publish_options(&self) -> TrackPublishOptions { + TrackPublishOptions { + video_codec: self.source.codec().into(), + video_encoder: VideoEncoderBackend::PreEncoded, + simulcast: false, + ..Default::default() + } + } + + /// Returns a cancellation handle for this pump. + pub fn stop_handle(&self) -> PumpStop { + self.stop.clone() + } + + /// Returns the underlying capture source. + pub fn source(&self) -> &S { + &self.source + } + + /// Runs the pump on the calling thread until the source ends, an error + /// occurs, or the stop handle fires. + /// + /// Sources block. On an async runtime, run this on a dedicated thread + /// (see [`EncodedVideoPump::spawn`]) or a blocking pool. + pub fn run(mut self) -> Result { + let mut frames_captured = 0; + let mut awaiting_initial_keyframe = true; + let exit = loop { + if self.stop.is_stopped() { + break PumpExit::Stopped; + } + if let Some(target) = self.rtc_source.take_rate_control_request() { + self.source.update_rate_control(target); + } + if self.rtc_source.take_keyframe_request() { + self.source.request_keyframe(); + } + + let Some(access_unit) = self.source.next_access_unit(&self.stop)? else { + // `None` is end of stream, unless the source returned early + // because the stop handle fired mid-wait. + break if self.stop.is_stopped() { + PumpExit::Stopped + } else { + PumpExit::EndOfStream + }; + }; + + // Drop pre-roll deltas: decoding can only start at a keyframe. + if awaiting_initial_keyframe && access_unit.frame_type != EncodedFrameType::Key { + continue; + } + awaiting_initial_keyframe = false; + + let metadata = self.frame_metadata.as_mut().and_then(|callback| callback(&access_unit)); + capture_access_unit(&self.rtc_source, &access_unit, metadata)?; + frames_captured += 1; + }; + Ok(PumpStats { frames_captured, exit }) + } + + /// Runs the pump on a dedicated thread. + /// + /// A panic on the pump thread is reported as [`PumpError::Panicked`] + /// when the pump is joined. + pub fn spawn(self) -> io::Result + where + S: 'static, + { + let stop = self.stop_handle(); + spawn_pump(stop, move || self.run()) + } +} + +impl fmt::Debug for EncodedVideoPump { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EncodedVideoPump") + .field("rtc_source", &self.rtc_source) + .field("stop", &self.stop) + .finish_non_exhaustive() + } +} + +fn capture_access_unit( + rtc_source: &NativeVideoSource, + access_unit: &OwnedEncodedAccessUnit, + frame_metadata: Option, +) -> Result<(), PumpError> { + // An empty payload is a violation of the source contract, so it is + // attributed to the source rather than the pump. + if access_unit.payload.is_empty() { + return Err(PumpError::Source(SourceError::new( + "source produced an access unit with an empty payload", + ))); + } + + let frame = EncodedVideoFrame { + codec: access_unit.codec.into(), + payload: &access_unit.payload, + timestamp_us: access_unit.timestamp_us, + frame_type: access_unit.frame_type.into(), + resolution: access_unit.resolution.into(), + frame_metadata, + }; + rtc_source.capture_encoded_frame(&frame).then_some(()).ok_or(PumpError::CaptureFailed) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use bytes::Bytes; + + use super::*; + use crate::{encoded::EncodedVideoCodec, error::SourceError, primitive::VideoResolution}; + + const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; + + struct FakeEncodedSource { + access_units: VecDeque, + } + + impl FakeEncodedSource { + fn new(access_units: impl IntoIterator) -> Self { + Self { access_units: access_units.into_iter().collect() } + } + } + + impl EncodedVideoSource for FakeEncodedSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn codec(&self) -> EncodedVideoCodec { + EncodedVideoCodec::VP8 + } + + fn next_access_unit( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { + Ok(self.access_units.pop_front()) + } + } + + fn access_unit(timestamp_us: i64, frame_type: EncodedFrameType) -> OwnedEncodedAccessUnit { + OwnedEncodedAccessUnit::new( + EncodedVideoCodec::VP8, + vec![1, 2, 3], + timestamp_us, + frame_type, + RESOLUTION, + ) + } + + #[test] + fn encoded_pump_starts_at_initial_keyframe() { + let source = FakeEncodedSource::new([ + access_unit(1, EncodedFrameType::Delta), + access_unit(2, EncodedFrameType::Delta), + access_unit(3, EncodedFrameType::Key), + access_unit(4, EncodedFrameType::Delta), + ]); + let stats = EncodedVideoPump::new(source).run().unwrap(); + assert_eq!(stats.frames_captured, 2); + } + + #[test] + fn boxed_source_drives_generic_pump() { + // The dynamic-instantiation pattern: box at the edge, same pump. + let source: Box = + Box::new(FakeEncodedSource::new([access_unit(1, EncodedFrameType::Key)])); + let pump = EncodedVideoPump::new(source); + assert_eq!(pump.publish_options().video_encoder, VideoEncoderBackend::PreEncoded); + let stats = pump.run().unwrap(); + assert_eq!(stats.frames_captured, 1); + } + + #[test] + fn metadata_callback_runs_per_captured_access_unit() { + let source = FakeEncodedSource::new([ + access_unit(1, EncodedFrameType::Delta), // dropped pre-roll, no callback + access_unit(2, EncodedFrameType::Key), + access_unit(3, EncodedFrameType::Delta), + ]); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let calls_in_callback = calls.clone(); + let stats = EncodedVideoPump::new(source) + .with_frame_metadata(move |access_unit| { + assert!(access_unit.timestamp_us > 1); + calls_in_callback.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + None + }) + .run() + .unwrap(); + assert_eq!(stats.frames_captured, 2); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 2); + } + + #[test] + fn encoded_pump_rejects_empty_payloads() { + let mut unit = access_unit(1, EncodedFrameType::Key); + unit.payload = Bytes::new(); + + let error = EncodedVideoPump::new(FakeEncodedSource::new([unit])).run().unwrap_err(); + assert!(matches!(&error, PumpError::Source(_))); + assert!(error.to_string().contains("empty payload")); + } + + #[test] + fn encoded_publish_options_use_passthrough() { + let pump = EncodedVideoPump::new(FakeEncodedSource::new([])); + let options = pump.publish_options(); + assert_eq!(options.video_encoder, VideoEncoderBackend::PreEncoded); + assert!(!options.simulcast); + } +} diff --git a/livekit-capture/src/error.rs b/livekit-capture/src/error.rs new file mode 100644 index 000000000..93803d83e --- /dev/null +++ b/livekit-capture/src/error.rs @@ -0,0 +1,50 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The error type shared by capture sources. +//! +//! Concrete errors live with what produces them: each source module defines +//! its own error type, and the pumps report through +//! [`PumpError`](crate::pump::PumpError). + +use std::{error::Error as StdError, fmt}; + +#[cfg(any(feature = "source-clock", feature = "source-pattern"))] +pub use crate::renderer::RendererError; + +/// Error returned by a capture source. +/// +/// `Display` and [`StdError::source`] delegate to the wrapped backend +/// error. +#[derive(Debug)] +pub struct SourceError(Box); + +impl SourceError { + /// Wraps a backend error. + pub fn new(error: impl Into>) -> Self { + Self(error.into()) + } +} + +impl fmt::Display for SourceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl StdError for SourceError { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + self.0.source() + } +} diff --git a/livekit-capture/src/lib.rs b/livekit-capture/src/lib.rs new file mode 100644 index 000000000..b1aef1220 --- /dev/null +++ b/livekit-capture/src/lib.rs @@ -0,0 +1,31 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Video capture for the LiveKit Rust SDK. +//! +//! A capture source produces video: pixel frames ([`pixel`]) or pre-encoded +//! access units ([`encoded`]). A pump drives a source and publishes its +//! output to an RTC video source. Ready-made sources live in [`sources`] +//! and can be enabled by their corresponding features. + +pub mod encoded; +pub mod error; +pub mod pixel; +pub mod primitive; +pub mod pump; +pub mod sources; + +#[cfg(any(feature = "source-clock", feature = "source-pattern"))] +mod renderer; +mod utils; diff --git a/livekit-capture/src/pixel/mod.rs b/livekit-capture/src/pixel/mod.rs new file mode 100644 index 000000000..8979e5c0e --- /dev/null +++ b/livekit-capture/src/pixel/mod.rs @@ -0,0 +1,67 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pixel (unencoded) video: the source trait and its pump. +//! +//! A source yields libwebrtc [`VideoFrame`](livekit::webrtc::video_frame::VideoFrame)s, +//! so any [`VideoBuffer`](livekit::webrtc::video_frame::VideoBuffer) — CPU +//! planes or platform-native — passes to the RTC source without an +//! intermediate copy. [`PixelVideoPump`] drives a source and publishes its +//! frames through the WebRTC encoder. +//! +//! [`PixelVideoSource`] is object-safe and implemented for `Box`, +//! so sources constructed dynamically run through the same generic pump. + +mod pump; + +use livekit::webrtc::video_frame::BoxVideoFrame; + +pub use pump::PixelVideoPump; + +use crate::{error::SourceError, primitive::VideoResolution, pump::PumpStop}; + +/// Source of pixel (unencoded) video frames, such as a camera device. +pub trait PixelVideoSource: Send { + /// Nominal output resolution, used to size the RTC source. + fn resolution(&self) -> VideoResolution; + + /// Blocks until the next frame is available. Returns `Ok(None)` at the + /// end of the stream. + /// + /// Implementations must return `Ok(None)` promptly once `stop` fires: + /// integrate the token into the blocking wait, or bound each wait to + /// about one frame interval. The pump uses the token to tell a stop + /// from the end of the stream. + /// + /// Implementations can pre-fill the frame's `frame_metadata`. A + /// metadata callback set on the pump takes precedence when it returns + /// `Some`. + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError>; +} + +impl PixelVideoSource for Box { + fn resolution(&self) -> VideoResolution { + (**self).resolution() + } + + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + (**self).next_frame(stop) + } +} + +// Object safety is part of this trait's contract: dynamic applications box +// sources at their edge and drive them through the same generic pumps. +const _: () = { + fn _assert_object_safe(_: &dyn PixelVideoSource) {} +}; diff --git a/livekit-capture/src/pixel/pump.rs b/livekit-capture/src/pixel/pump.rs new file mode 100644 index 000000000..97500b1e2 --- /dev/null +++ b/livekit-capture/src/pixel/pump.rs @@ -0,0 +1,306 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Pumps pixel frames from a capture source into an RTC video source. + +use crate::{ + pixel::PixelVideoSource, + pump::{spawn_pump, PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, +}; +use livekit::{ + options::TrackPublishOptions, + webrtc::{ + video_frame::{BoxVideoFrame, FrameMetadata}, + video_source::{native::NativeVideoSource, RtcVideoSource}, + }, +}; +use std::{fmt, io}; + +/// Callback that supplies packet-trailer metadata for a pixel frame. +type FrameMetadataFn = Box Option + Send>; + +/// Pumps a [`PixelVideoSource`] into an RTC video source and publishes its +/// frames through the WebRTC encoder. +pub struct PixelVideoPump { + source: S, + rtc_source: NativeVideoSource, + stop: PumpStop, + frame_metadata: Option, +} + +impl PixelVideoPump { + /// Creates a pump for a pixel source and builds the matching RTC source. + pub fn new(source: S) -> Self { + let rtc_source = NativeVideoSource::new(source.resolution().into(), false); + Self { source, rtc_source, stop: PumpStop::new(), frame_metadata: None } + } + + /// Sets a callback that supplies packet-trailer metadata for each frame. + /// + /// When the callback returns `Some`, it overrides metadata the source + /// pre-filled on the frame. Subscribers receive metadata only when the + /// matching [`TrackPublishOptions::frame_metadata_features`] are enabled + /// on the published track. + pub fn with_frame_metadata( + mut self, + frame_metadata: impl FnMut(&BoxVideoFrame) -> Option + Send + 'static, + ) -> Self { + self.frame_metadata = Some(Box::new(frame_metadata)); + self + } + + /// Returns the RTC source to create the local track with. + pub fn rtc_source(&self) -> RtcVideoSource { + RtcVideoSource::Native(self.rtc_source.clone()) + } + + /// Returns publish options for a pixel source. + pub fn publish_options(&self) -> TrackPublishOptions { + TrackPublishOptions::default() + } + + /// Returns a cancellation handle for this pump. + pub fn stop_handle(&self) -> PumpStop { + self.stop.clone() + } + + /// Returns the underlying capture source. + pub fn source(&self) -> &S { + &self.source + } + + /// Runs the pump on the calling thread until the source ends, an error + /// occurs, or the stop handle fires. + /// + /// Sources block. On an async runtime, run this on a dedicated thread + /// (see [`PixelVideoPump::spawn`]) or a blocking pool. + pub fn run(mut self) -> Result { + let mut frames_captured = 0; + let exit = loop { + if self.stop.is_stopped() { + break PumpExit::Stopped; + } + let Some(mut frame) = self.source.next_frame(&self.stop)? else { + // `None` is end of stream, unless the source returned early + // because the stop handle fired mid-wait. + break if self.stop.is_stopped() { + PumpExit::Stopped + } else { + PumpExit::EndOfStream + }; + }; + if let Some(metadata) = + self.frame_metadata.as_mut().and_then(|callback| callback(&frame)) + { + frame.frame_metadata = Some(metadata); + } + self.rtc_source.capture_frame(&frame); + frames_captured += 1; + }; + Ok(PumpStats { frames_captured, exit }) + } + + /// Runs the pump on a dedicated thread. + /// + /// A panic on the pump thread is reported as [`PumpError::Panicked`] + /// when the pump is joined. + pub fn spawn(self) -> io::Result + where + S: 'static, + { + let stop = self.stop_handle(); + spawn_pump(stop, move || self.run()) + } +} + +impl fmt::Debug for PixelVideoPump { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PixelVideoPump") + .field("rtc_source", &self.rtc_source) + .field("stop", &self.stop) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation}; + + use super::*; + use crate::{error::SourceError, primitive::VideoResolution}; + + const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; + + /// Pixel RTC sources spawn their keepalive task at construction; give the + /// tests the runtime context an SDK application would have. + fn runtime_context() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("failed to build test runtime") + } + + fn pixel_frame(timestamp_us: i64) -> BoxVideoFrame { + VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us, + frame_metadata: None, + buffer: Box::new(I420Buffer::new(RESOLUTION.width, RESOLUTION.height)), + } + } + + struct FakePixelSource { + frames: VecDeque, + } + + impl FakePixelSource { + fn new(frames: impl IntoIterator) -> Self { + Self { frames: frames.into_iter().collect() } + } + } + + impl PixelVideoSource for FakePixelSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame(&mut self, _stop: &PumpStop) -> Result, SourceError> { + Ok(self.frames.pop_front()) + } + } + + #[test] + fn pixel_pump_captures_all_frames_until_eof() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); + let stats = PixelVideoPump::new(source).run().unwrap(); + assert_eq!(stats.frames_captured, 3); + assert_eq!(stats.exit, PumpExit::EndOfStream); + } + + #[test] + fn boxed_source_drives_generic_pump() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + // The dynamic-instantiation pattern: box at the edge, same pump. + let source: Box = + Box::new(FakePixelSource::new([pixel_frame(1), pixel_frame(2)])); + let stats = PixelVideoPump::new(source).run().unwrap(); + assert_eq!(stats.frames_captured, 2); + } + + #[test] + fn metadata_callback_runs_per_frame() { + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let source = FakePixelSource::new([pixel_frame(1), pixel_frame(2), pixel_frame(3)]); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let calls_in_callback = calls.clone(); + let stats = PixelVideoPump::new(source) + .with_frame_metadata(move |frame| { + assert!(frame.timestamp_us > 0); + calls_in_callback.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + None + }) + .run() + .unwrap(); + assert_eq!(stats.frames_captured, 3); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 3); + } + + #[test] + fn pump_panics_become_errors() { + struct PanickingSource; + + impl PixelVideoSource for PanickingSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { + panic!("source exploded"); + } + } + + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let running = PixelVideoPump::new(PanickingSource).spawn().unwrap(); + let error = running.join().unwrap_err(); + assert!( + matches!(&error, PumpError::Panicked(message) if message.contains("source exploded")) + ); + } + + #[test] + fn running_pump_stops_on_signal() { + struct EndlessSource; + + impl PixelVideoSource for EndlessSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok(Some(pixel_frame(0))) + } + } + + let runtime = runtime_context(); + let _guard = runtime.enter(); + + let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(20)); + let stats = running.stop_and_join().unwrap(); + assert!(stats.frames_captured > 0); + assert_eq!(stats.exit, PumpExit::Stopped); + } + + #[tokio::test] + async fn pump_stops_and_joins_async() { + struct EndlessSource; + + impl PixelVideoSource for EndlessSource { + fn resolution(&self) -> VideoResolution { + RESOLUTION + } + + fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, SourceError> { + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok(Some(pixel_frame(0))) + } + } + + let running = PixelVideoPump::new(EndlessSource).spawn().unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let stats = running.stop_and_join_async().await.unwrap(); + assert!(stats.frames_captured > 0); + } +} diff --git a/livekit-capture/src/primitive.rs b/livekit-capture/src/primitive.rs new file mode 100644 index 000000000..4ed757b09 --- /dev/null +++ b/livekit-capture/src/primitive.rs @@ -0,0 +1,81 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Basic video types, such as [`VideoResolution`]. + +// TODO: in a future refactor, move these types into their own +// crate (e.g., `livekit-video-primitives`) so all crates in this workspace can work +// with common types without creating undesirable dependencies. + +/// Pixel dimensions of a video frame. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct VideoResolution { + /// Frame width in pixels. + pub width: u32, + /// Frame height in pixels. + pub height: u32, +} + +impl VideoResolution { + /// Creates a video resolution from a width and height in pixels. + /// + /// ``` + /// # use livekit_capture::primitive::VideoResolution; + /// let resolution = VideoResolution::new(1920, 1080); + /// assert_eq!(resolution.width, 1920); + /// assert_eq!(resolution.height, 1080); + /// ``` + pub const fn new(width: u32, height: u32) -> Self { + Self { width, height } + } + + /// Returns the width divided by the height, or `None` if the height is + /// zero. + /// + /// ``` + /// # use livekit_capture::primitive::VideoResolution; + /// assert_eq!(VideoResolution::new(1920, 960).aspect_ratio(), Some(2.0)); + /// assert_eq!(VideoResolution::new(1920, 0).aspect_ratio(), None); + /// ``` + pub fn aspect_ratio(&self) -> Option { + if self.height == 0 { + return None; + } + Some(f64::from(self.width) / f64::from(self.height)) + } +} + +impl std::fmt::Display for VideoResolution { + /// Formats as `WIDTHxHEIGHT`. + /// + /// ``` + /// # use livekit_capture::primitive::VideoResolution; + /// assert_eq!(VideoResolution::new(1920, 1080).to_string(), "1920x1080"); + /// ``` + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}x{}", self.width, self.height) + } +} + +impl From for livekit::webrtc::video_source::VideoResolution { + fn from(value: VideoResolution) -> Self { + Self { width: value.width, height: value.height } + } +} diff --git a/livekit-capture/src/pump.rs b/livekit-capture/src/pump.rs new file mode 100644 index 000000000..462f9ef65 --- /dev/null +++ b/livekit-capture/src/pump.rs @@ -0,0 +1,181 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Types shared by the capture pumps. +//! +//! The pumps live with their frame kinds: +//! [`PixelVideoPump`](crate::pixel::PixelVideoPump) pumps pixel frames, and +//! [`EncodedVideoPump`](crate::encoded::EncodedVideoPump) pumps encoded +//! access units. Both spawn into the same [`RunningPump`]. + +use crate::error::SourceError; +use std::{ + any::Any, + io, + panic::{catch_unwind, AssertUnwindSafe}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, +}; +use thiserror::Error; + +/// Error returned by a pump run. +#[derive(Debug, Error)] +pub enum PumpError { + /// The capture source failed or violated its contract. + #[error("capture source failed: {0}")] + Source(#[from] SourceError), + /// The RTC source rejected a frame. + #[error("capture source rejected the frame")] + CaptureFailed, + /// The pump thread panicked. + #[error("pump panicked: {0}")] + Panicked(String), +} + +/// Why a pump run ended successfully. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PumpExit { + /// The stop handle fired. + Stopped, + /// The source reached the end of its stream. + EndOfStream, +} + +/// Statistics returned when a pump run ends. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct PumpStats { + /// Number of frames or access units captured. + pub frames_captured: u64, + /// Why the run ended. + pub exit: PumpExit, +} + +/// Cancellation handle for a pump. +/// +/// The handle is cheap to clone. Call [`PumpStop::stop`] from any thread to +/// make the pump return after the frame in flight. +#[derive(Debug, Clone, Default)] +pub struct PumpStop(Arc); + +impl PumpStop { + /// Creates a new handle. + pub fn new() -> Self { + Self::default() + } + + /// Signals the pump to stop. + pub fn stop(&self) { + self.0.store(true, Ordering::Release); + } + + /// Returns whether [`PumpStop::stop`] was called. + pub fn is_stopped(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + +/// Spawns a pump run on a dedicated thread with panic capture and a +/// completion signal. +pub(crate) fn spawn_pump( + stop: PumpStop, + run: impl FnOnce() -> Result + Send + 'static, +) -> io::Result { + let (finished_tx, finished_rx) = tokio::sync::watch::channel(false); + + let thread = thread::Builder::new().name("lk-video-pump".to_owned()).spawn(move || { + let result = catch_unwind(AssertUnwindSafe(run)) + .unwrap_or_else(|panic| Err(PumpError::Panicked(panic_message(&*panic)))); + let _ = finished_tx.send(true); + result + })?; + + Ok(RunningPump { stop, thread, finished: finished_rx }) +} + +/// Renders a panic payload for [`PumpError::Panicked`]. +fn panic_message(panic: &(dyn Any + Send)) -> String { + if let Some(message) = panic.downcast_ref::<&str>() { + (*message).to_owned() + } else if let Some(message) = panic.downcast_ref::() { + message.clone() + } else { + "opaque panic payload".to_owned() + } +} + +/// A pump of either kind that runs on a dedicated thread. +/// +/// A stop takes effect between frames: a source that is blocked on its next +/// frame completes that wait before the pump observes the signal. +#[derive(Debug)] +pub struct RunningPump { + stop: PumpStop, + thread: thread::JoinHandle>, + // Flipped to true by the pump thread just before it exits. + finished: tokio::sync::watch::Receiver, +} + +impl RunningPump { + /// Returns a cancellation handle for the pump. + pub fn stop_handle(&self) -> PumpStop { + self.stop.clone() + } + + /// Signals the pump to stop after the frame in flight. + pub fn stop(&self) { + self.stop.stop(); + } + + /// Returns whether the pump thread exited. + pub fn is_finished(&self) -> bool { + self.thread.is_finished() + } + + /// Waits for the pump thread to exit. + /// + /// Panics on the pump thread are reported as [`PumpError::Panicked`]. + pub fn join(self) -> Result { + self.thread.join().unwrap_or_else(|panic| Err(PumpError::Panicked(panic_message(&*panic)))) + } + + /// Signals the pump to stop and waits for its thread to exit. + pub fn stop_and_join(self) -> Result { + self.stop(); + self.join() + } + + /// Waits for the pump thread to exit without blocking the async runtime. + /// + /// This works under any async runtime, not only tokio, and is safe to + /// hold across long waits — for example in a `select!` that supervises + /// every running pump. Panics on the pump thread are reported as + /// [`PumpError::Panicked`]. + pub async fn join_async(mut self) -> Result { + // An error means the sender dropped, which also implies the pump + // thread is done; either way the join below returns promptly. + let _ = self.finished.wait_for(|finished| *finished).await; + self.join() + } + + /// Signals the pump to stop and waits for its thread to exit without + /// blocking the async runtime. + pub async fn stop_and_join_async(self) -> Result { + self.stop(); + self.join_async().await + } +} diff --git a/livekit-capture/src/renderer.rs b/livekit-capture/src/renderer.rs new file mode 100644 index 000000000..a0b8da706 --- /dev/null +++ b/livekit-capture/src/renderer.rs @@ -0,0 +1,485 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Crate-internal GPU renderer shared by the shader-backed sources. +//! +//! [`ShaderRenderer`] renders a WGSL module offscreen through wgpu, one +//! frame at a time, and reads each frame back as I420. The caller +//! supplies the module and the per-frame uniform bytes. [`FramePacer`] +//! paces the frames against an ideal timeline. + +use crate::{primitive::VideoResolution, pump::PumpStop}; +use livekit::webrtc::video_frame::I420Buffer; +use std::{ + sync::{mpsc, Arc, Mutex}, + thread, + time::{Duration, Instant}, +}; +use thiserror::Error; + +/// Render target format. Its memory layout (B, G, R, A) is the layout +/// libyuv names ARGB. +const TARGET_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Bgra8Unorm; + +/// Bytes per pixel of [`TARGET_FORMAT`]. +const TARGET_BYTES_PER_PIXEL: u32 = 4; + +/// Upper bound on one blocking GPU wait, so the stop token is observed +/// promptly. +const STOP_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Total time to wait for one frame readback before the renderer fails. +const READBACK_TIMEOUT: Duration = Duration::from_secs(5); + +/// Error returned by the GPU renderer. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum RendererError { + /// No compatible GPU adapter is available. + #[error("no compatible GPU adapter: {0}")] + NoAdapter(String), + /// The GPU adapter rejected the device request. + #[error("failed to open the GPU device: {0}")] + Device(String), + /// The shader or its pipeline failed to build. + #[error("failed to build the shader pipeline: {0}")] + ShaderCompile(String), + /// The GPU reported an error. + #[error("GPU error: {0}")] + Backend(String), + /// Reading the rendered frame back from the GPU failed. + #[error("failed to read the frame back from the GPU: {0}")] + Readback(String), + /// Pixel conversion failed. + #[error("failed to convert the rendered frame to I420: {0}")] + Convert(&'static str), +} + +/// Paces frames against an ideal timeline, so frame timestamps are +/// jitter-free. +#[derive(Debug)] +pub(crate) struct FramePacer { + interval_us: u64, + started: Option, + frame_index: u64, +} + +impl FramePacer { + /// Creates a pacer. The frame rate must be non-zero. + pub(crate) fn new(framerate_fps: u32) -> Self { + let interval_us = (Duration::from_secs(1) / framerate_fps).as_micros() as u64; + Self { interval_us, started: None, frame_index: 0 } + } + + /// Sleeps until the next frame is due. Returns the elapsed time on + /// the ideal timeline and the index of the frame. + /// + /// The sleep is at most one frame interval. + pub(crate) fn wait_for_next_frame(&mut self) -> (Duration, u64) { + let started = *self.started.get_or_insert_with(Instant::now); + let elapsed = Duration::from_micros(self.frame_index.saturating_mul(self.interval_us)); + let due = started + elapsed; + if let Some(wait) = due.checked_duration_since(Instant::now()) { + thread::sleep(wait); + } + let frame_index = self.frame_index; + self.frame_index += 1; + (elapsed, frame_index) + } +} + +/// Renders a WGSL module offscreen and reads frames back as I420. +/// +/// The module must define a vertex entry point `vs_main` and a fragment +/// entry point `fs_main`. The renderer draws one triangle, which must +/// cover the full target. The module can declare one uniform buffer at +/// group 0, binding 0. The caller supplies its bytes for each frame. +pub(crate) struct ShaderRenderer { + device: wgpu::Device, + queue: wgpu::Queue, + pipeline: wgpu::RenderPipeline, + bind_group: wgpu::BindGroup, + uniform_buffer: wgpu::Buffer, + uniform_size: u64, + target: wgpu::Texture, + target_view: wgpu::TextureView, + /// Readback destination, reused across frames. Rows are padded to + /// the wgpu copy alignment. + staging: wgpu::Buffer, + padded_bytes_per_row: u32, + resolution: VideoResolution, + /// First uncaptured GPU error, stashed by the device error handler + /// and surfaced on the next frame. + device_error: Arc>>, +} + +impl ShaderRenderer { + /// Opens a GPU device, compiles the module, and builds the pipeline + /// and readback resources. + pub(crate) fn new( + resolution: VideoResolution, + module_code: &str, + uniform_size: u64, + ) -> Result { + let VideoResolution { width, height } = resolution; + let padded_bytes_per_row = padded_bytes_per_row(width) + .ok_or_else(|| RendererError::Backend("resolution is too large".to_owned()))?; + + // Rendering is offscreen, so no display handle is needed. WGPU_* + // environment variables can override the backend and adapter + // selection. + let instance = + wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::from_env().unwrap_or_default(), + ..Default::default() + })) + .map_err(|err| RendererError::NoAdapter(err.to_string()))?; + + let info = adapter.get_info(); + log::info!("Rendering with GPU \"{}\" ({})", info.name, info.backend); + + // Clamp the default limits to what the adapter supports, so weaker + // adapters (GL, software rasterizers) still open. A resolution + // beyond the clamped limits fails texture creation below. + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { + label: Some("lk_render_device"), + required_limits: wgpu::Limits::default().or_worse_values_from(&adapter.limits()), + ..Default::default() + })) + .map_err(|err| RendererError::Device(err.to_string()))?; + + // Runtime GPU errors have no return channel of their own: stash + // the first one and report it from the next render_frame call. + let device_error: Arc>> = Arc::default(); + let sink = Arc::clone(&device_error); + device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { + log::error!("render GPU error: {error}"); + let mut slot = sink.lock().unwrap(); + if slot.is_none() { + *slot = Some(error.to_string()); + } + })); + + // Compile the shader and build the pipeline under an error scope, + // so a bad shader fails construction with its compile message. + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("lk_render_module"), + source: wgpu::ShaderSource::Wgsl(module_code.into()), + }); + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("lk_render_bind_group_layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu::BufferSize::new(uniform_size), + }, + count: None, + }], + }); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("lk_render_pipeline_layout"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("lk_render_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format: TARGET_FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + multiview_mask: None, + cache: None, + }); + if let Some(error) = pollster::block_on(scope.pop()) { + return Err(RendererError::ShaderCompile(error.to_string())); + } + + // Build the target and readback resources under their own scope, + // so an unsupported resolution also fails construction. + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let target = device.create_texture(&wgpu::TextureDescriptor { + label: Some("lk_render_target"), + size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: TARGET_FORMAT, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let target_view = target.create_view(&wgpu::TextureViewDescriptor::default()); + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("lk_render_uniforms"), + size: uniform_size, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("lk_render_bind_group"), + layout: &bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }], + }); + let staging = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("lk_render_staging"), + size: u64::from(padded_bytes_per_row) * u64::from(height), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + if let Some(error) = pollster::block_on(scope.pop()) { + return Err(RendererError::Backend(error.to_string())); + } + + Ok(Self { + device, + queue, + pipeline, + bind_group, + uniform_buffer, + uniform_size, + target, + target_view, + staging, + padded_bytes_per_row, + resolution, + device_error, + }) + } + + /// Renders one frame with the given uniform bytes and reads it back + /// as I420. Returns `Ok(None)` when the stop token fires during the + /// readback wait. + /// + /// Every blocking wait is bounded by [`STOP_POLL_INTERVAL`], so the + /// stop token is observed promptly. + pub(crate) fn render_frame( + &self, + uniform: &[u8], + stop: &PumpStop, + ) -> Result, RendererError> { + debug_assert_eq!(uniform.len() as u64, self.uniform_size); + self.check_device_error()?; + + let VideoResolution { width, height } = self.resolution; + self.queue.write_buffer(&self.uniform_buffer, 0, uniform); + + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("lk_render") }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("lk_render_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.target_view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + ..Default::default() + }); + pass.set_pipeline(&self.pipeline); + pass.set_bind_group(0, &self.bind_group, &[]); + pass.draw(0..3, 0..1); + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.target, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &self.staging, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.padded_bytes_per_row), + rows_per_image: None, + }, + }, + wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + ); + // Schedule the mapping with the submission, so no separate + // map_async call is needed after submit. + let (mapped_tx, mapped_rx) = mpsc::channel(); + encoder.map_buffer_on_submit(&self.staging, wgpu::MapMode::Read, .., move |result| { + let _ = mapped_tx.send(result); + }); + let submission = self.queue.submit([encoder.finish()]); + + if !self.wait_for_map(submission, &mapped_rx, stop)? { + // Stopped: cancel the pending mapping to leave the buffer + // reusable. + self.staging.unmap(); + return Ok(None); + } + + let mapped = self.staging.slice(..).get_mapped_range(); + let converted = convert_to_i420(&mapped, self.padded_bytes_per_row, width, height); + drop(mapped); + self.staging.unmap(); + converted.map(Some) + } + + /// Waits for the staging buffer to be mapped. Returns `Ok(false)` when + /// the stop token fires first. + fn wait_for_map( + &self, + submission: wgpu::SubmissionIndex, + mapped: &mpsc::Receiver>, + stop: &PumpStop, + ) -> Result { + let deadline = Instant::now() + READBACK_TIMEOUT; + loop { + let poll = self.device.poll(wgpu::PollType::Wait { + submission_index: Some(submission.clone()), + timeout: Some(STOP_POLL_INTERVAL), + }); + match poll { + Ok(_) | Err(wgpu::PollError::Timeout) => {} + Err(err) => return Err(RendererError::Readback(err.to_string())), + } + match mapped.try_recv() { + Ok(Ok(())) => return Ok(true), + Ok(Err(err)) => return Err(RendererError::Readback(err.to_string())), + Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => { + return Err(RendererError::Readback("map callback was dropped".to_owned())); + } + } + self.check_device_error()?; + if stop.is_stopped() { + return Ok(false); + } + if Instant::now() >= deadline { + return Err(RendererError::Readback("timed out waiting for the GPU".to_owned())); + } + } + } + + /// Reports the first stashed GPU error, if there is one. + fn check_device_error(&self) -> Result<(), RendererError> { + match &*self.device_error.lock().unwrap() { + Some(message) => Err(RendererError::Backend(message.clone())), + None => Ok(()), + } + } +} + +/// Returns the staging-buffer row stride: the pixel row size rounded up +/// to the wgpu copy alignment. `None` when the value overflows `u32`. +fn padded_bytes_per_row(width: u32) -> Option { + let unpadded = u64::from(width) * u64::from(TARGET_BYTES_PER_PIXEL); + let align = u64::from(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT); + u32::try_from(unpadded.div_ceil(align) * align).ok() +} + +/// Returns whether a GPU adapter is available. Tests that need a GPU +/// skip when there is none. +#[cfg(test)] +pub(crate) fn gpu_available() -> bool { + let instance = + wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle_from_env()); + pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).is_ok() +} + +/// Converts one padded BGRA image to a freshly allocated I420 buffer. +fn convert_to_i420( + source: &[u8], + source_stride: u32, + width: u32, + height: u32, +) -> Result { + if source.len() < source_stride as usize * height as usize { + return Err(RendererError::Convert("mapped frame is too short")); + } + let source_stride = i32::try_from(source_stride) + .map_err(|_| RendererError::Convert("stride exceeds supported range"))?; + let width_i32 = i32::try_from(width) + .map_err(|_| RendererError::Convert("width exceeds supported range"))?; + let height_i32 = i32::try_from(height) + .map_err(|_| RendererError::Convert("height exceeds supported range"))?; + + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers `height` rows of `source_stride` bytes, and the + // destination planes come from a freshly allocated I420Buffer with matching width, + // height, and strides. + let ret = unsafe { + yuv_sys::rs_ARGBToI420( + source.as_ptr(), + source_stride, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + }; + if ret != 0 { + return Err(RendererError::Convert("ARGBToI420 failed")); + } + Ok(buffer) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rows_are_padded_to_the_copy_alignment() { + assert_eq!(padded_bytes_per_row(64), Some(256)); + assert_eq!(padded_bytes_per_row(321), Some(1536)); + assert_eq!(padded_bytes_per_row(1280), Some(5120)); + assert_eq!(padded_bytes_per_row(u32::MAX), None); + } + + #[test] + fn pacer_reports_the_ideal_timeline() { + let mut pacer = FramePacer::new(1000); + let (first_elapsed, first_index) = pacer.wait_for_next_frame(); + let (second_elapsed, second_index) = pacer.wait_for_next_frame(); + assert_eq!((first_elapsed.as_micros(), first_index), (0, 0)); + assert_eq!((second_elapsed.as_micros(), second_index), (1_000, 1)); + } +} diff --git a/livekit-capture/src/sources/clock.rs b/livekit-capture/src/sources/clock.rs new file mode 100644 index 000000000..7dce25dba --- /dev/null +++ b/livekit-capture/src/sources/clock.rs @@ -0,0 +1,272 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Wall-clock video source, for latency measurement. +//! +//! [`ClockVideoSource`] renders the local time as HH:MM:SS.mmm on the +//! GPU, with a grid below the digits that shows the milliseconds as +//! filled cells. The source samples the wall clock once per frame. +//! Rendering is offscreen through [wgpu], so the source needs no window +//! or display. +//! +//! The source reads each frame back from the GPU and converts it to I420 +//! on the CPU. +//! +//! [wgpu]: https://wgpu.rs + +use crate::{ + error::SourceError, + pixel::PixelVideoSource, + primitive::VideoResolution, + pump::PumpStop, + renderer::{FramePacer, RendererError, ShaderRenderer}, +}; +use chrono::Timelike; +use livekit::webrtc::video_frame::{BoxVideoFrame, VideoFrame, VideoRotation}; +use std::fmt; +use thiserror::Error; + +/// Complete WGSL module for the clock. +const CLOCK_SHADER: &str = include_str!("../../shaders/clock.wgsl"); + +/// Number of characters on the clock face: HH:MM:SS.mmm. +const CHAR_COUNT: usize = 12; + +/// Character codes for the separators. Codes 0 to 9 are digits. +const COLON: u32 = 10; +const DOT: u32 = 11; + +/// Size of the uniform block: `vec2` + padding + 3 * `vec4`. +const UNIFORM_BUFFER_SIZE: u64 = 64; + +/// Configuration for a [`ClockVideoSource`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct ClockVideoSourceConfig { + /// Output resolution. + pub resolution: VideoResolution, + /// Output frame rate in frames per second. + pub framerate_fps: u32, +} + +/// Error returned by a clock video source. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ClockVideoSourceError { + /// The configured resolution has a zero component. + #[error("clock source resolution must be non-zero")] + ZeroResolution, + /// The configured frame rate is zero. + #[error("clock source frame rate must be non-zero")] + ZeroFramerate, + /// The GPU renderer failed. + #[error(transparent)] + Render(#[from] RendererError), +} + +/// Pixel video source that renders a wall clock with millisecond +/// precision. +/// +/// The source sleeps to pace itself to the configured frame rate. It +/// never reaches the end of its stream — stop the pump that drives it +/// instead. +pub struct ClockVideoSource { + config: ClockVideoSourceConfig, + renderer: ShaderRenderer, + pacer: FramePacer, +} + +impl ClockVideoSource { + /// Creates the source. GPU setup runs on the tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`ClockVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: ClockVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await + } + + /// Selects a GPU adapter, compiles the clock shader, and builds the + /// render pipeline. + /// + /// Construction fails when no GPU is available, or for a zero + /// resolution or frame rate. + pub fn new_blocking(config: ClockVideoSourceConfig) -> Result { + validate_config(&config).map_err(SourceError::new)?; + let renderer = ShaderRenderer::new(config.resolution, CLOCK_SHADER, UNIFORM_BUFFER_SIZE) + .map_err(|error| SourceError::new(ClockVideoSourceError::Render(error)))?; + let pacer = FramePacer::new(config.framerate_fps); + Ok(Self { config, renderer, pacer }) + } + + /// Returns the configuration the source was created with. + pub fn config(&self) -> ClockVideoSourceConfig { + self.config + } +} + +impl PixelVideoSource for ClockVideoSource { + fn resolution(&self) -> VideoResolution { + self.config.resolution + } + + // The pacing sleep is at most one frame interval, and the renderer + // bounds every readback wait, so the stop token is observed promptly. + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + let (elapsed, _) = self.pacer.wait_for_next_frame(); + + // Sample the wall clock after the pacing sleep, so the shown + // time is as close as possible to the capture time. + let now = chrono::Local::now(); + let chars = + clock_chars(now.hour(), now.minute(), now.second(), now.nanosecond() / 1_000_000); + let uniform = uniform_bytes(self.config.resolution, &chars); + + let buffer = self + .renderer + .render_frame(&uniform, stop) + .map_err(|error| SourceError::new(ClockVideoSourceError::Render(error)))?; + let Some(buffer) = buffer else { + // The stop token fired during the readback wait. + return Ok(None); + }; + + Ok(Some(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: elapsed.as_micros() as i64, + frame_metadata: None, + buffer: Box::new(buffer), + })) + } +} + +impl fmt::Debug for ClockVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ClockVideoSource").field("config", &self.config).finish_non_exhaustive() + } +} + +/// Validates the CPU-checkable parts of a configuration. +fn validate_config(config: &ClockVideoSourceConfig) -> Result<(), ClockVideoSourceError> { + let VideoResolution { width, height } = config.resolution; + if width == 0 || height == 0 { + return Err(ClockVideoSourceError::ZeroResolution); + } + if config.framerate_fps == 0 { + return Err(ClockVideoSourceError::ZeroFramerate); + } + Ok(()) +} + +/// Returns the twelve character codes for HH:MM:SS.mmm. +fn clock_chars(hour: u32, minute: u32, second: u32, millisecond: u32) -> [u32; CHAR_COUNT] { + [ + (hour / 10) % 10, + hour % 10, + COLON, + (minute / 10) % 10, + minute % 10, + COLON, + (second / 10) % 10, + second % 10, + DOT, + (millisecond / 100) % 10, + (millisecond / 10) % 10, + millisecond % 10, + ] +} + +/// Serializes the uniform block: viewport size, padding, and the twelve +/// character codes at their 16-byte-aligned offset. +fn uniform_bytes( + resolution: VideoResolution, + chars: &[u32; CHAR_COUNT], +) -> [u8; UNIFORM_BUFFER_SIZE as usize] { + let mut bytes = [0u8; UNIFORM_BUFFER_SIZE as usize]; + bytes[0..4].copy_from_slice(&(resolution.width as f32).to_ne_bytes()); + bytes[4..8].copy_from_slice(&(resolution.height as f32).to_ne_bytes()); + for (index, code) in chars.iter().enumerate() { + let offset = 16 + index * 4; + bytes[offset..offset + 4].copy_from_slice(&code.to_ne_bytes()); + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::renderer::gpu_available; + + fn test_config() -> ClockVideoSourceConfig { + ClockVideoSourceConfig { + resolution: VideoResolution { width: 320, height: 180 }, + framerate_fps: 1000, + } + } + + #[test] + fn clock_chars_render_three_millisecond_digits() { + assert_eq!(clock_chars(12, 34, 56, 789), [1, 2, COLON, 3, 4, COLON, 5, 6, DOT, 7, 8, 9]); + } + + #[test] + fn validation_rejects_zero_resolution_and_framerate() { + let mut config = test_config(); + config.resolution = VideoResolution::new(0, 180); + assert!(matches!(validate_config(&config), Err(ClockVideoSourceError::ZeroResolution))); + + let mut config = test_config(); + config.framerate_fps = 0; + assert!(matches!(validate_config(&config), Err(ClockVideoSourceError::ZeroFramerate))); + } + + #[test] + fn uniform_places_chars_at_their_alignment() { + let chars = clock_chars(12, 34, 56, 789); + let bytes = uniform_bytes(VideoResolution::new(1280, 720), &chars); + assert_eq!(f32::from_ne_bytes(bytes[0..4].try_into().unwrap()), 1280.0); + assert_eq!(f32::from_ne_bytes(bytes[4..8].try_into().unwrap()), 720.0); + assert_eq!(u32::from_ne_bytes(bytes[16..20].try_into().unwrap()), 1); + assert_eq!(u32::from_ne_bytes(bytes[60..64].try_into().unwrap()), 9); + } + + #[test] + fn clock_renders_frames() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut source = ClockVideoSource::new_blocking(test_config()).unwrap(); + + let stop = PumpStop::new(); + let first = source.next_frame(&stop).unwrap().unwrap(); + let second = source.next_frame(&stop).unwrap().unwrap(); + assert_eq!((first.buffer.width(), first.buffer.height()), (320, 180)); + assert_eq!(first.timestamp_us, 0); + assert_eq!(second.timestamp_us, 1_000); + + // The clock is centered with a margin, so the top-left pixel is + // background black, and the lit digits stand out well above it. + let i420 = first.buffer.as_i420().expect("clock source yields I420 buffers"); + let (y, _, _) = i420.data(); + assert!(y[0] <= 20, "top-left pixel is not background: {}", y[0]); + let lit = y.iter().filter(|&&luma| luma > 100).count(); + assert!(lit > 20, "no clock pixels found (lit count {lit})"); + } +} diff --git a/livekit-capture/src/sources/device/avfoundation.rs b/livekit-capture/src/sources/device/avfoundation.rs new file mode 100644 index 000000000..4fa6eded2 --- /dev/null +++ b/livekit-capture/src/sources/device/avfoundation.rs @@ -0,0 +1,1542 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! macOS device capture backend built on AVFoundation. +//! +//! This module is an implementation detail of [`super::DeviceVideoSource`]: +//! nothing AVFoundation-specific leaves it. Frames are delivered as native +//! IOSurface-backed `CVPixelBuffer`s when the negotiated session supports +//! that (full-range NV12 without software scaling), and converted to I420 +//! otherwise. + +use std::ffi::c_void; +use std::ops::Deref; +use std::ptr::NonNull; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +use dispatch2::{DispatchQueue, DispatchRetained}; +use livekit::webrtc::video_frame::{ + native::NativeBuffer, BoxVideoFrame, I420Buffer, VideoBuffer, VideoFrame, VideoRotation, +}; +use objc2::rc::Retained; +use objc2::runtime::{AnyObject, ProtocolObject}; +use objc2::{define_class, msg_send, sel, AnyThread, DefinedClass, Message}; +use objc2_av_foundation::{ + AVCaptureDevice, AVCaptureDeviceFormat, AVCaptureDeviceInput, AVCaptureOutput, + AVCaptureSession, AVCaptureSessionPreset1280x720, AVCaptureSessionPreset1920x1080, + AVCaptureSessionPreset640x480, AVCaptureSessionPresetHigh, AVCaptureSessionPresetInputPriority, + AVCaptureSessionPresetMedium, AVCaptureVideoDataOutput, + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureVideoStabilizationMode, + AVMediaTypeVideo, +}; +use objc2_core_media::{ + CMClock, CMSampleBuffer, CMTime, CMTimeFlags, CMVideoFormatDescriptionGetDimensions, +}; +use objc2_core_video::{ + kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferMetalCompatibilityKey, + kCVPixelBufferPixelFormatTypeKey, kCVPixelFormatType_32BGRA, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, + kCVPixelFormatType_420YpCbCr8PlanarFullRange, kCVPixelFormatType_422YpCbCr8, + kCVPixelFormatType_422YpCbCr8FullRange, kCVPixelFormatType_422YpCbCr8_yuvs, kCVReturnSuccess, + CVImageBuffer, CVPixelBuffer, CVPixelBufferGetBaseAddress, CVPixelBufferGetBaseAddressOfPlane, + CVPixelBufferGetBytesPerRow, CVPixelBufferGetBytesPerRowOfPlane, CVPixelBufferGetHeight, + CVPixelBufferGetHeightOfPlane, CVPixelBufferGetPixelFormatType, CVPixelBufferGetPlaneCount, + CVPixelBufferGetWidth, CVPixelBufferGetWidthOfPlane, CVPixelBufferLockBaseAddress, + CVPixelBufferLockFlags, CVPixelBufferUnlockBaseAddress, +}; +use objc2_foundation::{NSDictionary, NSNumber, NSObject, NSObjectProtocol, NSString}; + +use super::timestamp::{ + elapsed_us, unix_time_us_now, validate_capture_timestamp_us, MAX_CAPTURE_TIMESTAMP_AGE_US, +}; +use super::{ + capture_frame_metadata, DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, + DeviceSelector, DeviceVideoSourceConfig, DeviceVideoSourceError, +}; +use crate::{primitive::VideoResolution, pump::PumpStop}; + +unsafe extern "C" { + fn CFRelease(cf: *const c_void); + fn CVPixelBufferGetIOSurface(pixel_buffer: *const CVPixelBuffer) -> *const c_void; +} + +/// How long session construction waits for the device's first frame, which +/// establishes the delivered format. +const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long one frame wait may block before the stop token is rechecked. +const STOP_CHECK_INTERVAL: Duration = Duration::from_millis(100); + +/// Returns whether the backend can request this frame format from a device. +fn is_supported_request_format(frame_format: DeviceFrameFormat) -> bool { + matches!( + frame_format, + DeviceFrameFormat::Nv12 | DeviceFrameFormat::Bgra | DeviceFrameFormat::I420 + ) +} + +/// AVFoundation capture session satisfying the backend contract. +pub(super) struct Session { + format: DeviceFormat, + target_resolution: Option, + native_frame_supported: bool, + inner: SessionInner, +} + +// SAFETY: `Session` owns AVFoundation objects and only exposes `&mut self` +// frame capture plus `Drop`; moving ownership to another thread does not +// create concurrent access to those Objective-C objects. +unsafe impl Send for Session {} + +impl Session { + /// Opens a capture session and waits for the first frame to establish + /// the delivered format. + pub(super) fn open(config: &DeviceVideoSourceConfig) -> Result { + super::validate_config(config, is_supported_request_format)?; + + let inner = SessionInner::new(config)?; + let initial_frame = inner.wait_for_format(FIRST_FRAME_TIMEOUT)?; + inner.discard_pending_frame(); + let mut format = initial_frame.format; + format.framerate_fps = requested_framerate(&config.format).unwrap_or(30); + let target_resolution = requested_output_resolution(&config.format, format.resolution); + if let Some(resolution) = target_resolution { + format.resolution = resolution; + } + let session = Self { + format, + target_resolution, + native_frame_supported: initial_frame.native_frame_supported, + inner, + }; + log::info!( + "Opened device \"{}\" ({}): {} ({})", + session.inner.device_name, + session.inner.device_id, + session.format, + if session.native_capture() { "native buffers" } else { "converted to I420" }, + ); + Ok(session) + } + + /// Returns the negotiated capture format. + pub(super) fn format(&self) -> DeviceFormat { + self.format + } + + fn native_capture(&self) -> bool { + self.native_frame_supported + && self.target_resolution.is_none() + && self.format.frame_format == DeviceFrameFormat::Nv12 + } + + /// Blocks until the next frame is available, returning `Ok(None)` once + /// the stop token fires. + pub(super) fn next_frame( + &mut self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + // Convert only after the frame queue's mutex is released: conversion + // locks the pixel buffer and runs a full-frame libyuv copy, and + // holding the mutex through that would block `push_frame` on the + // AVFoundation delegate queue, which drops camera frames while + // stalled (`setAlwaysDiscardsLateVideoFrames(true)`). + let Some(queued) = self.inner.wait_take_queued_frame(stop)? else { + return Ok(None); + }; + + if self.native_capture() { + return queued.into_native_frame().map(|frame| Some(box_frame(frame))); + } + + let mut frame = queued.into_i420_frame()?; + if let Some(resolution) = self.target_resolution { + if frame.buffer.width() != resolution.width + || frame.buffer.height() != resolution.height + { + let width = i32::try_from(resolution.width).map_err(|_| { + DeviceVideoSourceError::InvalidFrame("scaled width exceeds i32") + })?; + let height = i32::try_from(resolution.height).map_err(|_| { + DeviceVideoSourceError::InvalidFrame("scaled height exceeds i32") + })?; + frame.buffer = frame.buffer.scale(width, height); + } + } + Ok(Some(box_frame(frame))) + } +} + +/// Type-erases a concrete frame for the pixel source contract. +fn box_frame + 'static>( + frame: VideoFrame, +) -> BoxVideoFrame { + VideoFrame { + rotation: frame.rotation, + timestamp_us: frame.timestamp_us, + frame_metadata: frame.frame_metadata, + buffer: Box::new(frame.buffer), + } +} + +/// Lists AVFoundation video capture devices. +pub(super) fn devices() -> Result, DeviceVideoSourceError> { + // SAFETY: AVMediaTypeVideo is a framework-provided immutable NSString + // constant. We only borrow it to ask AVFoundation for video devices. + let media_type = unsafe { AVMediaTypeVideo }.ok_or(DeviceVideoSourceError::DeviceNotFound)?; + // SAFETY: AVFoundation returns an immutable NSArray of currently available + // AVCaptureDevice instances. We only retain/copy string properties from it. + #[allow(deprecated)] + let devices = unsafe { AVCaptureDevice::devicesWithMediaType(media_type) }; + + let mut results = Vec::with_capacity(devices.len()); + for device in devices.iter() { + // SAFETY: These Objective-C property getters return retained NSStrings + // for a live AVCaptureDevice from the immutable devices array. + let id = unsafe { device.uniqueID() }.to_string(); + let name = unsafe { device.localizedName() }.to_string(); + let model_id = non_empty_string(unsafe { device.modelID() }.to_string()); + let manufacturer = non_empty_string(unsafe { device.manufacturer() }.to_string()); + + results.push(DeviceInfo { + id, + name, + model_id, + manufacturer, + formats: Vec::new(), + formats_complete: false, + }); + } + + Ok(results) +} + +fn non_empty_string(value: String) -> Option { + (!value.is_empty()).then_some(value) +} + +fn requested_output_resolution( + request: &DeviceFormatRequest, + delivered: VideoResolution, +) -> Option { + let DeviceFormatRequest::Closest(format) = request else { + return None; + }; + if format.resolution == delivered { + return None; + } + (resolution_area(format.resolution) <= resolution_area(delivered)).then_some(format.resolution) +} + +fn resolution_area(resolution: VideoResolution) -> u64 { + resolution.width as u64 * resolution.height as u64 +} + +struct SessionInner { + session: Retained, + _input: Retained, + output: Retained, + _delegate: Retained, + _queue: DispatchRetained, + shared: Arc, + device_name: String, + device_id: String, +} + +impl Drop for SessionInner { + fn drop(&mut self) { + self.shared.stop(); + // SAFETY: The output and session are owned by this wrapper. Clearing + // the delegate before stopping prevents callbacks from racing with + // the delegate being released during teardown. + unsafe { + self.output.setSampleBufferDelegate_queue(None, None); + self.session.stopRunning(); + } + } +} + +impl SessionInner { + fn new(config: &DeviceVideoSourceConfig) -> Result { + let device = select_device(&config.device)?; + // SAFETY: These property getters return retained NSStrings for a + // live AVCaptureDevice. + let device_name = unsafe { device.localizedName() }.to_string(); + let device_id = unsafe { device.uniqueID() }.to_string(); + let session = unsafe { AVCaptureSession::new() }; + let input = unsafe { AVCaptureDeviceInput::deviceInputWithDevice_error(&device) }.map_err( + |err| DeviceVideoSourceError::Backend(err.localizedDescription().to_string()), + )?; + let output = unsafe { AVCaptureVideoDataOutput::new() }; + let shared = Arc::new(FrameQueue::default()); + let delegate = CaptureDelegate::new(shared.clone()); + let queue = DispatchQueue::new("io.livekit.capture.device", None); + let active_format = select_active_format(&device, &config.format)?; + + // SAFETY: The session is newly created and not running. We add a + // camera input and video data output only after canAdd* checks. + unsafe { + session.beginConfiguration(); + session.setAutomaticallyConfiguresCaptureDeviceForWideColor(false); + if active_format.is_none() { + if let Some(preset) = session_preset(&config.format) { + session.setSessionPreset(preset); + } + } + let config_result = (|| { + if !session.canAddInput(&input) { + return Err(DeviceVideoSourceError::Backend( + "capture device input could not be added".to_string(), + )); + } + session.addInput(&input); + + configure_device(&device, &config.format, active_format.as_deref())?; + if active_format.is_some() + && session.canSetSessionPreset(AVCaptureSessionPresetInputPriority) + { + session.setSessionPreset(AVCaptureSessionPresetInputPriority); + } + configure_input_frame_duration(&input, &device, &config.format); + + if let Some(video_settings) = preferred_video_settings(&output) { + output.setVideoSettings(Some(&video_settings)); + } + output.setAlwaysDiscardsLateVideoFrames(true); + output.setSampleBufferDelegate_queue( + Some(ProtocolObject::from_ref(&*delegate)), + Some(&queue), + ); + if !session.canAddOutput(&output) { + return Err(DeviceVideoSourceError::Backend( + "video data output could not be added".to_string(), + )); + } + session.addOutput(&output); + configure_output_connection(&output)?; + Ok(()) + })(); + session.commitConfiguration(); + config_result?; + } + + // SAFETY: Configuration has been committed and the session is ready + // to synchronously start delivering video samples. + unsafe { + session.startRunning(); + } + + Ok(Self { + session, + _input: input, + output, + _delegate: delegate, + _queue: queue, + shared, + device_name, + device_id, + }) + } + + fn wait_for_format( + &self, + timeout: Duration, + ) -> Result { + self.shared.wait_for_format(timeout) + } + + fn wait_take_queued_frame( + &self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + self.shared.wait_take_queued_frame(stop) + } + + fn discard_pending_frame(&self) { + self.shared.discard_latest(); + } +} + +fn preferred_video_settings( + output: &AVCaptureVideoDataOutput, +) -> Option>> { + let preferred = [ + // WebRTC's VideoToolbox H.264 encoder allocates full-range NV12 + // buffers for its CPU upload path. Prefer the same CoreVideo + // format for direct CVPixelBuffer input so the native path does + // not have to reset VideoToolbox into a separate video-range pool. + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + ]; + // SAFETY: `output` is a live AVCaptureVideoDataOutput owned by the session setup path, and + // querying advertised CV pixel formats does not mutate Rust-managed memory. + let supported_formats = unsafe { output.availableVideoCVPixelFormatTypes() }; + let pixel_format_type = preferred + .into_iter() + .find(|preferred| supported_formats.iter().any(|format| format.as_u32() == *preferred))?; + + let pixel_format = NSNumber::new_u32(pixel_format_type); + let metal_compatible = NSNumber::new_bool(true); + let iosurface_properties = NSDictionary::::new(); + // SAFETY: The CoreVideo constants are immutable CFString keys. + // `CFString` and `NSString` are toll-free bridged, which + // objc2-foundation exposes through `AsRef`. + let pixel_format_key: &NSString = unsafe { kCVPixelBufferPixelFormatTypeKey }.as_ref(); + // SAFETY: Same as above. + let iosurface_key: &NSString = unsafe { kCVPixelBufferIOSurfacePropertiesKey }.as_ref(); + // SAFETY: Same as above. + let metal_key: &NSString = unsafe { kCVPixelBufferMetalCompatibilityKey }.as_ref(); + Some(NSDictionary::from_slices( + &[pixel_format_key, iosurface_key, metal_key], + &[pixel_format.as_ref(), iosurface_properties.as_ref(), metal_compatible.as_ref()], + )) +} + +fn configure_input_frame_duration( + input: &AVCaptureDeviceInput, + device: &AVCaptureDevice, + request: &DeviceFormatRequest, +) { + let Some(framerate) = requested_framerate(request).filter(|framerate| *framerate > 0) else { + return; + }; + + // AVCaptureDeviceInput's locked-frame-duration API is macOS 26.0+, while + // the SDK builds against an older deployment target. Sending a selector the + // running OS does not implement raises an Objective-C exception, which Rust + // cannot catch and which therefore aborts the process, so probe first. + if !input.respondsToSelector(sel!(isLockedVideoFrameDurationSupported)) + || !input.respondsToSelector(sel!(setActiveLockedVideoFrameDuration:)) + { + return; + } + + // SAFETY: `input` is the live input just added to the session, and the + // selector was confirmed present above. + if !unsafe { input.isLockedVideoFrameDurationSupported() } { + return; + } + + // SAFETY: `device` and `input` belong to the same session setup path, and + // reading activeFormat is valid once the input has been added. + let duration = unsafe { device_format_frame_duration(&device.activeFormat(), framerate) }; + let Some(duration) = duration else { + return; + }; + + // SAFETY: `input` reports locked frame duration support, and `duration` + // came from a frame-rate range of the device's active format. + unsafe { + input.setActiveLockedVideoFrameDuration(duration); + } +} + +fn configure_output_connection( + output: &AVCaptureVideoDataOutput, +) -> Result<(), DeviceVideoSourceError> { + let media_type = unsafe { AVMediaTypeVideo }.ok_or(DeviceVideoSourceError::DeviceNotFound)?; + // SAFETY: `output` has just been added to a configured session. Querying + // its video connection does not mutate Rust-managed memory. + let Some(connection) = (unsafe { output.connectionWithMediaType(media_type) }) else { + return Err(DeviceVideoSourceError::Backend( + "video data output connection was not created".to_string(), + )); + }; + + // Keep frame-duration control on the device/input path. The deprecated + // output connection frame-duration setters can change whether macOS + // delivers IOSurface-backed pixel buffers. + // SAFETY: The connection is the video data output connection. Each + // setter is guarded by the corresponding support/configuration checks + // required by AVFoundation's API contract. + unsafe { + if connection.isVideoStabilizationSupported() { + connection.setPreferredVideoStabilizationMode(AVCaptureVideoStabilizationMode::Off); + } + if connection.automaticallyAdjustsVideoMirroring() { + connection.setAutomaticallyAdjustsVideoMirroring(false); + } + if connection.isVideoMirroringSupported() && connection.isVideoMirrored() { + connection.setVideoMirrored(false); + } + } + Ok(()) +} + +#[derive(Debug)] +struct CaptureDelegateIvars { + shared: Arc, +} + +define_class!( + // SAFETY: + // - The superclass NSObject does not have subclassing requirements. + // - CaptureDelegate does not implement Drop; retained Rust state lives in ivars. + #[unsafe(super = NSObject)] + #[thread_kind = AnyThread] + #[ivars = CaptureDelegateIvars] + struct CaptureDelegate; + + // SAFETY: `NSObjectProtocol` has no additional safety requirements. + unsafe impl NSObjectProtocol for CaptureDelegate {} + + // SAFETY: The selector signatures match the generated AVFoundation protocol. + unsafe impl AVCaptureVideoDataOutputSampleBufferDelegate for CaptureDelegate { + #[unsafe(method(captureOutput:didOutputSampleBuffer:fromConnection:))] + #[allow(non_snake_case)] + unsafe fn captureOutput_didOutputSampleBuffer_fromConnection( + &self, + _output: &AVCaptureOutput, + sample_buffer: &CMSampleBuffer, + _connection: &objc2_av_foundation::AVCaptureConnection, + ) { + if let Err(err) = process_sample_buffer(sample_buffer, &self.ivars().shared) { + self.ivars().shared.set_error(err.to_string()); + } + } + } +); + +impl CaptureDelegate { + fn new(shared: Arc) -> Retained { + let this = Self::alloc().set_ivars(CaptureDelegateIvars { shared }); + // SAFETY: `this` is freshly allocated and initialized exactly once + // using NSObject's designated initializer. + unsafe { msg_send![super(this), init] } + } +} + +/// Latest-frame mailbox shared between the AVFoundation delegate queue and +/// the capturing thread. +#[derive(Debug)] +struct FrameQueue { + state: Mutex, + ready: Condvar, + started_at: Instant, +} + +impl Default for FrameQueue { + fn default() -> Self { + Self { + state: Mutex::new(FrameQueueState::default()), + ready: Condvar::new(), + started_at: Instant::now(), + } + } +} + +#[derive(Debug, Default)] +struct FrameQueueState { + latest: Option, + stopped: bool, + error: Option, +} + +#[derive(Debug)] +struct InitialFrameInfo { + format: DeviceFormat, + native_frame_supported: bool, +} + +impl FrameQueue { + fn push_frame(&self, frame: QueuedFrame) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + if state.stopped { + return; + } + state.latest = Some(frame); + self.ready.notify_one(); + } + + fn set_error(&self, error: String) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + state.error = Some(error); + self.ready.notify_all(); + } + + /// Signals session teardown and wakes every blocked frame wait. + /// + /// Stopping is idempotent. `push_frame` discards frames delivered after + /// this point. + fn stop(&self) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + state.stopped = true; + self.ready.notify_all(); + } + + fn discard_latest(&self) { + let mut state = self.state.lock().expect("device frame queue poisoned"); + state.latest = None; + } + + fn wait_for_format( + &self, + timeout: Duration, + ) -> Result { + let deadline = Instant::now() + timeout; + let mut state = self.state.lock().expect("device frame queue poisoned"); + loop { + if let Some(frame) = state.latest.as_ref() { + return Ok(InitialFrameInfo { + format: DeviceFormat::new( + VideoResolution::new(frame.width, frame.height), + 0, + frame.source_format, + ), + native_frame_supported: frame.native_frame_supported(), + }); + } + if let Some(error) = state.error.take() { + return Err(DeviceVideoSourceError::Backend(error)); + } + if state.stopped { + return Err(DeviceVideoSourceError::Backend( + "capture session stopped before delivering a frame".to_string(), + )); + } + + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(DeviceVideoSourceError::FrameTimeout); + }; + let (next_state, _) = + self.ready.wait_timeout(state, remaining).expect("device frame queue poisoned"); + state = next_state; + } + } + + /// Blocks until a frame, a delegate error, or a stop arrives and moves + /// the frame out of the shared state, returning `Ok(None)` on stop. + /// + /// The state mutex guard is dropped when this returns, so callers convert + /// the fully owned frame without holding the lock. Each wait is bounded + /// by [`STOP_CHECK_INTERVAL`] so the stop token is observed promptly even + /// when the device stalls without delivering frames or errors. + fn wait_take_queued_frame( + &self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + let mut state = self.state.lock().expect("device frame queue poisoned"); + loop { + if let Some(frame) = state.latest.take() { + return Ok(Some(frame)); + } + if let Some(error) = state.error.take() { + return Err(DeviceVideoSourceError::Backend(error)); + } + if state.stopped || stop.is_stopped() { + return Ok(None); + } + let (next_state, _) = self + .ready + .wait_timeout(state, STOP_CHECK_INTERVAL) + .expect("device frame queue poisoned"); + state = next_state; + } + } + + fn timestamp_us(&self) -> i64 { + elapsed_us(self.started_at.elapsed()) + } +} + +#[derive(Debug)] +struct QueuedFrame { + pixel_buffer: RetainedPixelBuffer, + width: u32, + height: u32, + source_format: DeviceFrameFormat, + core_video_pixel_format: u32, + // Wall-clock capture time: the validated sensor timestamp when + // AVFoundation reports one, the read time otherwise. + capture_wall_time_us: u64, + timestamp_us: i64, + is_iosurface_backed: bool, +} + +impl QueuedFrame { + fn into_i420_frame(self) -> Result, DeviceVideoSourceError> { + let buffer = convert_pixel_buffer(self.pixel_buffer.as_ref())?; + Ok(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: self.timestamp_us, + frame_metadata: Some(capture_frame_metadata(self.capture_wall_time_us)), + buffer, + }) + } + + fn into_native_frame(self) -> Result, DeviceVideoSourceError> { + if !self.native_frame_supported() { + return Err(DeviceVideoSourceError::Backend( + "native capture requires an IOSurface-backed full-range NV12 buffer".to_string(), + )); + } + + let timestamp_us = self.timestamp_us; + let capture_wall_time_us = self.capture_wall_time_us; + let buffer = self.pixel_buffer.into_native_buffer(); + Ok(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us, + frame_metadata: Some(capture_frame_metadata(capture_wall_time_us)), + buffer, + }) + } + + fn native_frame_supported(&self) -> bool { + self.source_format == DeviceFrameFormat::Nv12 + && self.core_video_pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + && self.is_iosurface_backed + } +} + +fn pixel_buffer_has_iosurface(pixel_buffer: &CVPixelBuffer) -> bool { + // SAFETY: `pixel_buffer` is a valid CVPixelBufferRef. CoreVideo returns + // an unretained IOSurfaceRef; this code only checks for null and does + // not store or release the returned pointer. + !unsafe { CVPixelBufferGetIOSurface(pixel_buffer) }.is_null() +} + +#[derive(Debug)] +struct RetainedPixelBuffer { + ptr: NonNull, +} + +// SAFETY: `RetainedPixelBuffer` owns a +1 CoreFoundation reference to a +// CVPixelBuffer. CoreFoundation retain/release and CoreVideo pixel-buffer +// inspection are thread-safe for this usage, and mutable pixel access still +// goes through CoreVideo's lock/unlock API. +unsafe impl Send for RetainedPixelBuffer {} +// SAFETY: The wrapper exposes only shared access to the pixel buffer and +// releases its retained reference on drop. +unsafe impl Sync for RetainedPixelBuffer {} + +impl RetainedPixelBuffer { + fn from_image_buffer(image_buffer: T) -> Self + where + T: Deref, + { + let ptr = NonNull::from(&*image_buffer).cast::(); + std::mem::forget(image_buffer); + Self { ptr } + } + + fn as_ref(&self) -> &CVPixelBuffer { + // SAFETY: `ptr` was created from a retained CVImageBuffer returned + // by CMSampleBufferGetImageBuffer and remains valid until this + // wrapper drops or transfers that retain. + unsafe { self.ptr.as_ref() } + } + + fn into_native_buffer(self) -> NativeBuffer { + let ptr = self.ptr.as_ptr().cast::(); + std::mem::forget(self); + // SAFETY: `ptr` is a valid retained CVPixelBufferRef. The WebRTC + // bridge wraps it in RTCCVPixelBuffer and then releases the +1 + // retain we transfer here, so Rust must not release it afterwards. + unsafe { NativeBuffer::from_cv_pixel_buffer(ptr) } + } +} + +impl Drop for RetainedPixelBuffer { + fn drop(&mut self) { + // SAFETY: `ptr` owns one CoreFoundation retain unless ownership was + // transferred by `into_native_buffer`, which forgets `self`. + unsafe { CFRelease(self.ptr.as_ptr().cast::()) }; + } +} + +fn select_device( + selector: &DeviceSelector, +) -> Result, DeviceVideoSourceError> { + let media_type = unsafe { AVMediaTypeVideo }.ok_or(DeviceVideoSourceError::DeviceNotFound)?; + match selector { + DeviceSelector::Default => { + unsafe { AVCaptureDevice::defaultDeviceWithMediaType(media_type) } + .ok_or(DeviceVideoSourceError::DeviceNotFound) + } + DeviceSelector::Index(index) => { + #[allow(deprecated)] + let devices = unsafe { AVCaptureDevice::devicesWithMediaType(media_type) }; + devices + .iter() + .nth(*index) + .map(|device| device.retain()) + .ok_or(DeviceVideoSourceError::DeviceNotFound) + } + DeviceSelector::Id(id) => { + let id = NSString::from_str(id); + unsafe { AVCaptureDevice::deviceWithUniqueID(&id) } + .ok_or(DeviceVideoSourceError::DeviceNotFound) + } + } +} + +fn select_active_format( + device: &AVCaptureDevice, + request: &DeviceFormatRequest, +) -> Result>, DeviceVideoSourceError> { + match request { + DeviceFormatRequest::Default => Ok(None), + DeviceFormatRequest::Exact(format) => { + let selected = best_device_format( + device, + Some(format.resolution), + Some(format.framerate_fps), + SelectionMode::Exact, + ); + selected.map(Some).ok_or(DeviceVideoSourceError::UnsupportedFormat(*format)) + } + DeviceFormatRequest::Closest(format) => Ok(best_device_format( + device, + Some(format.resolution), + Some(format.framerate_fps), + SelectionMode::Closest, + )), + DeviceFormatRequest::HighestFramerate { resolution, .. } => { + Ok(best_device_format(device, *resolution, None, SelectionMode::HighestFramerate)) + } + DeviceFormatRequest::HighestResolution { framerate_fps, .. } => { + Ok(best_device_format(device, None, *framerate_fps, SelectionMode::HighestResolution)) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SelectionMode { + Exact, + Closest, + HighestFramerate, + HighestResolution, +} + +#[derive(Debug)] +struct DeviceFormatCandidate { + format: Retained, + resolution: VideoResolution, + framerate_supported: bool, + max_framerate: u32, +} + +fn best_device_format( + device: &AVCaptureDevice, + resolution: Option, + framerate: Option, + mode: SelectionMode, +) -> Option> { + // SAFETY: The AVCaptureDevice is retained for the session setup path; querying the + // immutable list of supported formats does not mutate Rust-managed memory. + let formats = unsafe { device.formats() }; + let mut candidates = formats + .iter() + .filter_map(|format| { + let candidate_resolution = device_format_resolution(&format)?; + let framerate_supported = framerate + .map(|framerate| device_format_supports_framerate(&format, framerate)) + .unwrap_or(true); + Some(DeviceFormatCandidate { + format: format.retain(), + resolution: candidate_resolution, + framerate_supported, + max_framerate: device_format_max_framerate(&format), + }) + }) + .collect::>(); + + if let Some(resolution) = resolution { + if mode == SelectionMode::Exact { + return candidates + .into_iter() + .find(|candidate| { + candidate.resolution == resolution && candidate.framerate_supported + }) + .map(|candidate| candidate.format); + } + } + + if framerate.is_some() && candidates.iter().any(|candidate| candidate.framerate_supported) { + candidates.retain(|candidate| candidate.framerate_supported); + } + + match mode { + SelectionMode::Exact => None, + SelectionMode::Closest => { + let resolution = resolution?; + candidates + .into_iter() + .min_by_key(|candidate| resolution_distance(candidate.resolution, resolution)) + .map(|candidate| candidate.format) + } + SelectionMode::HighestFramerate => candidates + .into_iter() + .filter(|candidate| { + resolution.map(|resolution| candidate.resolution == resolution).unwrap_or(true) + }) + .max_by_key(|candidate| { + ( + candidate.max_framerate, + candidate.resolution.width as u64 * candidate.resolution.height as u64, + ) + }) + .map(|candidate| candidate.format), + SelectionMode::HighestResolution => candidates + .into_iter() + .max_by_key(|candidate| { + ( + candidate.resolution.width as u64 * candidate.resolution.height as u64, + candidate.max_framerate, + ) + }) + .map(|candidate| candidate.format), + } +} + +fn device_format_resolution(format: &AVCaptureDeviceFormat) -> Option { + // SAFETY: `format` is an AVCaptureDeviceFormat from the device's immutable formats array. + // Its format description is a valid CMVideoFormatDescription for video capture formats. + let description = unsafe { format.formatDescription() }; + // SAFETY: `description` is the video format description returned by AVFoundation. + let dimensions = unsafe { CMVideoFormatDescriptionGetDimensions(&description) }; + if dimensions.width <= 0 || dimensions.height <= 0 { + return None; + } + Some(VideoResolution::new(dimensions.width as u32, dimensions.height as u32)) +} + +fn device_format_supports_framerate(format: &AVCaptureDeviceFormat, framerate: u32) -> bool { + device_format_frame_duration(format, framerate).is_some() +} + +/// Frame duration to apply for `framerate` on `format`, or `None` when no +/// frame-rate range covers it. +/// +/// The duration is taken from the matched range's own bounds instead of being +/// derived from `framerate` alone. AVFoundation raises an Objective-C +/// exception — which aborts the process, since Rust cannot catch it — for any +/// duration outside a range's `[minFrameDuration, maxFrameDuration]`, and +/// devices commonly advertise near-integral rates whose exact duration is not +/// the reciprocal of the rounded rate. A UVC camera reporting 30.00003 fps +/// accepts 1/30.00003 s but rejects 1/30 s. +fn device_format_frame_duration(format: &AVCaptureDeviceFormat, framerate: u32) -> Option { + let requested = framerate as f64; + // SAFETY: `format` is an AVCaptureDeviceFormat from the device's immutable formats array. + // The returned frame-rate ranges are immutable AVFoundation objects. + unsafe { format.videoSupportedFrameRateRanges() }.iter().find_map(|range| { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + let min = unsafe { range.minFrameRate() }; + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + let max = unsafe { range.maxFrameRate() }; + if requested < min.floor() || requested > max.ceil() { + return None; + } + // Rate and duration are inverses, so the slowest rate carries the + // longest duration. Snapping to an endpoint keeps a rounded request + // inside the bounds the format actually accepts. + Some(if requested <= min { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + unsafe { range.maxFrameDuration() } + } else if requested >= max { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + unsafe { range.minFrameDuration() } + } else { + // The rate lies strictly inside the range, so its reciprocal lies + // strictly inside the range's duration bounds. + // SAFETY: `requested` is finite and greater than zero here. + unsafe { CMTime::with_seconds(1.0 / requested, 600) } + }) + }) +} + +fn device_format_max_framerate(format: &AVCaptureDeviceFormat) -> u32 { + // SAFETY: `format` is an AVCaptureDeviceFormat from the device's immutable formats array. + // The returned frame-rate ranges are immutable AVFoundation objects. + unsafe { format.videoSupportedFrameRateRanges() } + .iter() + .map(|range| { + // SAFETY: AVFrameRateRange values are immutable for the lifetime of the object. + unsafe { range.maxFrameRate() }.floor().max(0.0) as u32 + }) + .max() + .unwrap_or_default() +} + +fn resolution_distance(actual: VideoResolution, requested: VideoResolution) -> u64 { + let width_delta = actual.width.abs_diff(requested.width) as u64; + let height_delta = actual.height.abs_diff(requested.height) as u64; + let pixel_delta = (actual.width as u64 * actual.height as u64) + .abs_diff(requested.width as u64 * requested.height as u64); + pixel_delta + width_delta * width_delta + height_delta * height_delta +} + +fn configure_device( + device: &AVCaptureDevice, + request: &DeviceFormatRequest, + active_format: Option<&AVCaptureDeviceFormat>, +) -> Result<(), DeviceVideoSourceError> { + let framerate = requested_framerate(request); + if active_format.is_none() && framerate.is_none() { + return Ok(()); + } + + unsafe { device.lockForConfiguration() } + .map_err(|err| DeviceVideoSourceError::Backend(err.localizedDescription().to_string()))?; + + let configure_result = configure_locked_device(device, active_format, framerate); + // SAFETY: The device was successfully locked above and must be unlocked exactly once. + unsafe { + device.unlockForConfiguration(); + } + configure_result +} + +fn configure_locked_device( + device: &AVCaptureDevice, + active_format: Option<&AVCaptureDeviceFormat>, + framerate: Option, +) -> Result<(), DeviceVideoSourceError> { + // SAFETY: The caller holds the AVCaptureDevice configuration lock, and `active_format` + // was selected from this device's formats array. + unsafe { + if let Some(active_format) = active_format { + device.setActiveFormat(active_format); + } + } + configure_low_latency_device_processing(device); + + let Some(framerate) = framerate.filter(|framerate| *framerate > 0) else { + return Ok(()); + }; + + let active_format = match active_format { + Some(active_format) => active_format.retain(), + // SAFETY: The caller holds the configuration lock, and reading activeFormat is valid. + None => unsafe { device.activeFormat() }, + }; + let Some(duration) = device_format_frame_duration(&active_format, framerate) else { + return Ok(()); + }; + + // SAFETY: The device is locked for configuration and `duration` came from a + // frame-rate range of the format now active on the device. + unsafe { + device.setActiveVideoMinFrameDuration(duration); + device.setActiveVideoMaxFrameDuration(duration); + } + Ok(()) +} + +fn configure_low_latency_device_processing(device: &AVCaptureDevice) { + // SAFETY: The caller holds the AVCaptureDevice configuration lock. + // Setters are guarded by their support/current-state predicates where + // AVFoundation requires that. + unsafe { + if device.automaticallyAdjustsVideoHDREnabled() { + device.setAutomaticallyAdjustsVideoHDREnabled(false); + } + if device.isVideoHDREnabled() { + device.setVideoHDREnabled(false); + } + if device.isLowLightBoostSupported() + && device.automaticallyEnablesLowLightBoostWhenAvailable() + { + device.setAutomaticallyEnablesLowLightBoostWhenAvailable(false); + } + if device.isSmoothAutoFocusSupported() && device.isSmoothAutoFocusEnabled() { + device.setSmoothAutoFocusEnabled(false); + } + } +} + +fn requested_framerate(request: &DeviceFormatRequest) -> Option { + match request { + DeviceFormatRequest::Default => None, + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + Some(format.framerate_fps) + } + DeviceFormatRequest::HighestFramerate { .. } => None, + DeviceFormatRequest::HighestResolution { framerate_fps, .. } => *framerate_fps, + } +} + +fn session_preset( + request: &DeviceFormatRequest, +) -> Option<&'static objc2_av_foundation::AVCaptureSessionPreset> { + let resolution = match request { + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + Some(format.resolution) + } + DeviceFormatRequest::HighestFramerate { resolution, .. } => *resolution, + DeviceFormatRequest::Default | DeviceFormatRequest::HighestResolution { .. } => None, + }?; + + exact_session_preset(resolution).or(Some(unsafe { AVCaptureSessionPresetHigh })) +} + +fn exact_session_preset( + resolution: VideoResolution, +) -> Option<&'static objc2_av_foundation::AVCaptureSessionPreset> { + match (resolution.width, resolution.height) { + (1920, 1080) => Some(unsafe { AVCaptureSessionPreset1920x1080 }), + (1280, 720) => Some(unsafe { AVCaptureSessionPreset1280x720 }), + (640, 480) => Some(unsafe { AVCaptureSessionPreset640x480 }), + (w, h) if w <= 640 && h <= 480 => Some(unsafe { AVCaptureSessionPresetMedium }), + _ => None, + } +} + +fn process_sample_buffer( + sample_buffer: &CMSampleBuffer, + shared: &FrameQueue, +) -> Result<(), DeviceVideoSourceError> { + let read_wall_time_us = unix_time_us_now().unwrap_or_default(); + let sensor_timestamp_us = sample_buffer_capture_wall_time_us(sample_buffer, read_wall_time_us); + let image_buffer = unsafe { sample_buffer.image_buffer() } + .ok_or(DeviceVideoSourceError::InvalidFrame("sample buffer has no image buffer"))?; + let pixel_buffer = RetainedPixelBuffer::from_image_buffer(image_buffer); + let pixel_buffer_ref = pixel_buffer.as_ref(); + let width = u32::try_from(CVPixelBufferGetWidth(pixel_buffer_ref)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("width is out of range"))?; + let height = u32::try_from(CVPixelBufferGetHeight(pixel_buffer_ref)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("height is out of range"))?; + let core_video_pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer_ref); + let source_format = frame_format_from_core_video(core_video_pixel_format)?; + let is_iosurface_backed = pixel_buffer_has_iosurface(pixel_buffer_ref); + + let capture_wall_time_us = sensor_timestamp_us.unwrap_or(read_wall_time_us); + shared.push_frame(QueuedFrame { + pixel_buffer, + width, + height, + source_format, + core_video_pixel_format, + capture_wall_time_us, + timestamp_us: shared.timestamp_us(), + is_iosurface_backed, + }); + Ok(()) +} + +fn sample_buffer_capture_wall_time_us( + sample_buffer: &CMSampleBuffer, + read_wall_time_us: u64, +) -> Option { + let sample_time = unsafe { sample_buffer.presentation_time_stamp() }; + + let timestamp_us = cm_time_to_us(sample_time)?; + if validate_capture_timestamp_us(timestamp_us, read_wall_time_us).is_some() { + return Some(timestamp_us); + } + + let host_now_us = current_host_time_us()?; + let age_us = host_now_us.checked_sub(timestamp_us)?; + if age_us > MAX_CAPTURE_TIMESTAMP_AGE_US { + return None; + } + read_wall_time_us.checked_sub(age_us) +} + +fn current_host_time_us() -> Option { + // SAFETY: The CoreMedia host time clock is a process-wide singleton and + // reading it does not mutate Rust-managed memory. + let host_clock = unsafe { CMClock::host_time_clock() }; + // SAFETY: `host_clock` is a valid retained CoreMedia clock. + let host_time = unsafe { host_clock.time() }; + cm_time_to_us(host_time) +} + +fn cm_time_to_us(time: CMTime) -> Option { + let flags = time.flags; + if !flags.contains(CMTimeFlags::Valid) || flags.intersects(CMTimeFlags::ImpliedValueFlagsMask) { + return None; + } + + // SAFETY: `time` is a valid CMTime value returned by CoreMedia. Invalid + // and indefinite values were filtered above. + let seconds = unsafe { time.seconds() }; + if !seconds.is_finite() || seconds < 0.0 { + return None; + } + + let micros = seconds * 1_000_000.0; + (micros <= u64::MAX as f64).then_some(micros.round() as u64) +} + +fn convert_pixel_buffer( + pixel_buffer: &CVPixelBuffer, +) -> Result { + let lock_flags = CVPixelBufferLockFlags::ReadOnly; + let lock_result = unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, lock_flags) }; + if lock_result != kCVReturnSuccess { + return Err(DeviceVideoSourceError::InvalidFrame("CVPixelBuffer lock failed")); + } + + let result = convert_locked_pixel_buffer(pixel_buffer); + + // SAFETY: The pixel buffer was locked above with the same flags. + let unlock_result = unsafe { CVPixelBufferUnlockBaseAddress(pixel_buffer, lock_flags) }; + if unlock_result != kCVReturnSuccess { + return Err(DeviceVideoSourceError::InvalidFrame("CVPixelBuffer unlock failed")); + } + + result +} + +fn convert_locked_pixel_buffer( + pixel_buffer: &CVPixelBuffer, +) -> Result { + let width = u32::try_from(CVPixelBufferGetWidth(pixel_buffer)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("width is out of range"))?; + let height = u32::try_from(CVPixelBufferGetHeight(pixel_buffer)) + .map_err(|_| DeviceVideoSourceError::InvalidFrame("height is out of range"))?; + let source_format = + frame_format_from_core_video(CVPixelBufferGetPixelFormatType(pixel_buffer))?; + + match source_format { + DeviceFrameFormat::Nv12 => convert_nv12(pixel_buffer, width, height), + DeviceFrameFormat::Bgra => convert_bgra(pixel_buffer, width, height), + DeviceFrameFormat::I420 => convert_i420(pixel_buffer, width, height), + DeviceFrameFormat::Uyvy => convert_uyvy(pixel_buffer, width, height), + DeviceFrameFormat::Yuyv => convert_yuy2(pixel_buffer, width, height), + other => Err(DeviceVideoSourceError::UnsupportedFrameFormat(other)), + } +} + +fn frame_format_from_core_video( + pixel_format: u32, +) -> Result { + match pixel_format { + format + if format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + || format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange => + { + Ok(DeviceFrameFormat::Nv12) + } + format if format == kCVPixelFormatType_32BGRA => Ok(DeviceFrameFormat::Bgra), + format + if format == kCVPixelFormatType_420YpCbCr8Planar + || format == kCVPixelFormatType_420YpCbCr8PlanarFullRange => + { + Ok(DeviceFrameFormat::I420) + } + format if format == kCVPixelFormatType_422YpCbCr8 => Ok(DeviceFrameFormat::Uyvy), + format + if format == kCVPixelFormatType_422YpCbCr8_yuvs + || format == kCVPixelFormatType_422YpCbCr8FullRange => + { + Ok(DeviceFrameFormat::Yuyv) + } + other => Err(DeviceVideoSourceError::Backend(format!( + "unsupported CoreVideo pixel format 0x{other:08x}" + ))), + } +} + +fn convert_nv12( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + if CVPixelBufferGetPlaneCount(pixel_buffer) < 2 { + return Err(DeviceVideoSourceError::InvalidFrame("NV12 buffer has fewer than two planes")); + } + + let y = plane(pixel_buffer, 0)?; + let uv = plane(pixel_buffer, 1)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slices cover the locked CVPixelBuffer planes for the duration of this + // call, and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_NV12ToI420( + y.data.as_ptr(), + y.stride as i32, + uv.data.as_ptr(), + uv.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("NV12ToI420 failed")); + } + Ok(buffer) +} + +fn convert_bgra( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + let bgra = packed_plane(pixel_buffer, 4)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers the locked CVPixelBuffer for the duration of this call, + // and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_BGRAToI420( + bgra.data.as_ptr(), + bgra.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("BGRAToI420 failed")); + } + Ok(buffer) +} + +fn convert_uyvy( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + let uyvy = packed_plane(pixel_buffer, 2)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers the locked CVPixelBuffer for the duration of this call, + // and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_UYVYToI420( + uyvy.data.as_ptr(), + uyvy.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("UYVYToI420 failed")); + } + Ok(buffer) +} + +fn convert_yuy2( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + let yuy2 = packed_plane(pixel_buffer, 2)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slice covers the locked CVPixelBuffer for the duration of this call, + // and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_YUY2ToI420( + yuy2.data.as_ptr(), + yuy2.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("YUY2ToI420 failed")); + } + Ok(buffer) +} + +fn convert_i420( + pixel_buffer: &CVPixelBuffer, + width: u32, + height: u32, +) -> Result { + if CVPixelBufferGetPlaneCount(pixel_buffer) < 3 { + return Err(DeviceVideoSourceError::InvalidFrame( + "I420 buffer has fewer than three planes", + )); + } + + let y = plane(pixel_buffer, 0)?; + let u = plane(pixel_buffer, 1)?; + let v = plane(pixel_buffer, 2)?; + let mut buffer = I420Buffer::new(width, height); + let (stride_y, stride_u, stride_v) = buffer.strides(); + let (dst_y, dst_u, dst_v) = buffer.data_mut(); + // SAFETY: The source slices cover the locked CVPixelBuffer planes for the duration of this + // call, and the destination planes come from a freshly allocated I420Buffer with matching + // width, height, and strides. + let ret = unsafe { + yuv_sys::rs_I420Copy( + y.data.as_ptr(), + y.stride as i32, + u.data.as_ptr(), + u.stride as i32, + v.data.as_ptr(), + v.stride as i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ) + }; + if ret != 0 { + return Err(DeviceVideoSourceError::Convert("I420Copy failed")); + } + Ok(buffer) +} + +struct Plane<'a> { + data: &'a [u8], + stride: usize, +} + +fn plane(pixel_buffer: &CVPixelBuffer, index: usize) -> Result, DeviceVideoSourceError> { + let plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + if index >= plane_count { + return Err(DeviceVideoSourceError::InvalidFrame("plane index is out of range")); + } + + let base = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, index); + if base.is_null() { + return Err(DeviceVideoSourceError::InvalidFrame("pixel plane has no base address")); + } + let stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, index); + let height = CVPixelBufferGetHeightOfPlane(pixel_buffer, index); + let width = CVPixelBufferGetWidthOfPlane(pixel_buffer, index); + let min_len = stride + .checked_mul(height.saturating_sub(1)) + .and_then(|value| value.checked_add(width)) + .ok_or(DeviceVideoSourceError::InvalidFrame("pixel plane size overflow"))?; + + // SAFETY: The CVPixelBuffer is locked for read-only access, the plane + // base address is non-null, and CoreVideo reports the minimum readable + // extent for this plane. + let data = unsafe { std::slice::from_raw_parts(base.cast::(), min_len) }; + Ok(Plane { data, stride }) +} + +fn packed_plane( + pixel_buffer: &CVPixelBuffer, + bytes_per_pixel: usize, +) -> Result, DeviceVideoSourceError> { + let base = CVPixelBufferGetBaseAddress(pixel_buffer); + if base.is_null() { + return Err(DeviceVideoSourceError::InvalidFrame("pixel buffer has no base address")); + } + let stride = CVPixelBufferGetBytesPerRow(pixel_buffer); + let height = CVPixelBufferGetHeight(pixel_buffer); + let width = CVPixelBufferGetWidth(pixel_buffer) + .checked_mul(bytes_per_pixel) + .ok_or(DeviceVideoSourceError::InvalidFrame("packed pixel row size overflow"))?; + let min_len = stride + .checked_mul(height.saturating_sub(1)) + .and_then(|value| value.checked_add(width)) + .ok_or(DeviceVideoSourceError::InvalidFrame("packed pixel buffer size overflow"))?; + + // SAFETY: The CVPixelBuffer is locked for read-only access, the base + // address is non-null, and CoreVideo reports the minimum readable extent + // for this packed buffer. + let data = unsafe { std::slice::from_raw_parts(base.cast::(), min_len) }; + Ok(Plane { data, stride }) +} + +#[cfg(test)] +mod tests { + use std::sync::{mpsc, Arc}; + use std::time::Duration; + + use super::{FrameQueue, STOP_CHECK_INTERVAL}; + use crate::pump::PumpStop; + + /// Upper bound on how long a woken frame wait may take to return before + /// the test declares the stop path broken. + const STOP_WAIT_TIMEOUT: Duration = Duration::from_secs(5); + + // `FrameQueue` is pure Rust state, so these tests run on macOS CI hosts + // without camera hardware or AVFoundation involvement. + + #[test] + fn stop_token_unblocks_frame_wait() { + let queue = Arc::new(FrameQueue::default()); + let stop = PumpStop::new(); + + let (done_tx, done_rx) = mpsc::channel(); + let stop_in_waiter = stop.clone(); + let queue_in_waiter = Arc::clone(&queue); + let waiter = std::thread::spawn(move || { + let result = queue_in_waiter.wait_take_queued_frame(&stop_in_waiter); + let _ = done_tx.send(()); + result + }); + + // Give the waiter time to block. There is no race if the stop lands + // first: the wait loop re-checks the token at least every + // STOP_CHECK_INTERVAL. + std::thread::sleep(Duration::from_millis(50)); + stop.stop(); + + done_rx + .recv_timeout(STOP_WAIT_TIMEOUT + STOP_CHECK_INTERVAL) + .expect("frame wait did not return after the stop token fired"); + let result = waiter.join().expect("frame wait thread panicked"); + assert!(matches!(result, Ok(None)), "unexpected frame wait result: {result:?}"); + } + + #[test] + fn frame_waits_return_none_once_queue_stopped() { + let queue = FrameQueue::default(); + queue.stop(); + // Stopping is idempotent. + queue.stop(); + + assert!(matches!(queue.wait_take_queued_frame(&PumpStop::new()), Ok(None))); + } + + #[test] + fn delegate_errors_surface_from_frame_wait() { + let queue = FrameQueue::default(); + queue.set_error("camera unplugged".to_string()); + + let error = queue + .wait_take_queued_frame(&PumpStop::new()) + .expect_err("delegate error must surface"); + assert!(error.to_string().contains("camera unplugged")); + } +} diff --git a/livekit-capture/src/sources/device/mod.rs b/livekit-capture/src/sources/device/mod.rs new file mode 100644 index 000000000..29173f7b4 --- /dev/null +++ b/livekit-capture/src/sources/device/mod.rs @@ -0,0 +1,610 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Camera device capture. +//! +//! [`DeviceVideoSource`] captures pixel frames from a video device through +//! the platform's native capture stack. Configuration, enumeration +//! ([`devices`]), and errors use one platform-neutral vocabulary. On +//! platforms without a backend the module still compiles, and construction +//! and enumeration fail with +//! [`DeviceVideoSourceError::UnsupportedPlatform`]. +//! +//! Where the platform supports it, frames reach the RTC track as +//! platform-native buffers without a CPU copy. Otherwise they are converted +//! to I420. + +#[cfg(target_os = "macos")] +mod avfoundation; +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod timestamp; +#[cfg(target_os = "linux")] +mod v4l2; + +#[cfg(target_os = "macos")] +use avfoundation as backend; +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +use unsupported as backend; +#[cfg(target_os = "linux")] +use v4l2 as backend; + +use std::fmt; + +use livekit::webrtc::video_frame::BoxVideoFrame; +use thiserror::Error; + +use crate::{ + error::SourceError, pixel::PixelVideoSource, primitive::VideoResolution, pump::PumpStop, +}; + +/// Selects the video device a [`DeviceVideoSource`] captures from. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "lowercase") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum DeviceSelector { + /// The platform default video device. + #[default] + Default, + /// The device at this position in the platform enumeration order. + Index(usize), + /// The device with this identifier, as reported by [`DeviceInfo::id`]. + Id(String), +} + +/// Frame format delivered by a capture device. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "lowercase") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum DeviceFrameFormat { + /// Planar I420/YUV420P. + I420, + /// Biplanar NV12. + Nv12, + /// Packed BGRA. + Bgra, + /// Packed RGB24. + Rgb24, + /// Packed BGR24. + Bgr24, + /// Packed YUYV/YUY2. + Yuyv, + /// Packed UYVY. + Uyvy, + /// Single-plane 8-bit luma. + Grey, + /// Encoded MJPEG frames. + Mjpeg, +} + +impl DeviceFrameFormat { + /// Returns a stable lower-case frame-format name. + pub const fn as_str(self) -> &'static str { + match self { + Self::I420 => "i420", + Self::Nv12 => "nv12", + Self::Bgra => "bgra", + Self::Rgb24 => "rgb24", + Self::Bgr24 => "bgr24", + Self::Yuyv => "yuyv", + Self::Uyvy => "uyvy", + Self::Grey => "grey", + Self::Mjpeg => "mjpeg", + } + } +} + +impl fmt::Display for DeviceFrameFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for DeviceFrameFormat { + type Err = DeviceFrameFormatParseError; + + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "i420" | "yuv420p" => Ok(Self::I420), + "nv12" => Ok(Self::Nv12), + "bgra" => Ok(Self::Bgra), + "rgb24" | "rgb" => Ok(Self::Rgb24), + "bgr24" | "bgr" => Ok(Self::Bgr24), + "yuyv" | "yuy2" => Ok(Self::Yuyv), + "uyvy" => Ok(Self::Uyvy), + "grey" | "greyscale" => Ok(Self::Grey), + "mjpeg" | "mjpg" => Ok(Self::Mjpeg), + _ => Err(DeviceFrameFormatParseError), + } + } +} + +/// Error returned when parsing a [`DeviceFrameFormat`] from a string. +#[derive(Debug, Clone, Copy, Error, PartialEq, Eq)] +#[error("unknown device frame format")] +pub struct DeviceFrameFormatParseError; + +/// Capture format offered by a device. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeviceFormat { + /// Frame dimensions. + pub resolution: VideoResolution, + /// Frame rate in frames per second. + pub framerate_fps: u32, + /// Frame format. + pub frame_format: DeviceFrameFormat, +} + +impl DeviceFormat { + /// Creates a device capture format. + pub const fn new( + resolution: VideoResolution, + framerate_fps: u32, + frame_format: DeviceFrameFormat, + ) -> Self { + Self { resolution, framerate_fps, frame_format } + } +} + +impl fmt::Display for DeviceFormat { + /// Formats as `WIDTHxHEIGHT@FPSfps FORMAT`. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}@{}fps {}", self.resolution, self.framerate_fps, self.frame_format) + } +} + +/// Format selection requested from a capture device. +/// +/// The device negotiates the delivered format, and +/// [`DeviceVideoSource::format`] reports the outcome. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "snake_case") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum DeviceFormatRequest { + /// Let the device choose its default format. + #[default] + Default, + /// Require an exact format match. + Exact(DeviceFormat), + /// Use the device's closest supported format. + Closest(DeviceFormat), + /// Prefer the highest frame rate, optionally constrained by resolution + /// and frame format. + HighestFramerate { + /// Optional resolution constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + resolution: Option, + /// Optional frame format constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + frame_format: Option, + }, + /// Prefer the highest resolution, optionally constrained by frame rate + /// and frame format. + HighestResolution { + /// Optional frame-rate constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + framerate_fps: Option, + /// Optional frame format constraint. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + frame_format: Option, + }, +} + +/// Video capture device discovered by [`devices`]. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeviceInfo { + /// Device identifier, usable with [`DeviceSelector::Id`]. + pub id: String, + /// Human-readable device name. + pub name: String, + /// Device model identifier, when available. + pub model_id: Option, + /// Device manufacturer, when available. + pub manufacturer: Option, + /// Capture formats reported by the device. + pub formats: Vec, + /// Whether [`DeviceInfo::formats`] is a complete list. Some platforms do + /// not enumerate formats up front. + pub formats_complete: bool, +} + +impl DeviceInfo { + /// Returns the selector that reopens this exact device. + pub fn selector(&self) -> DeviceSelector { + DeviceSelector::Id(self.id.clone()) + } +} + +/// Lists the video capture devices on this machine. +/// +/// Requires a running tokio runtime: enumeration runs on the tokio blocking +/// pool. Use [`devices_blocking`] outside of async contexts. +#[cfg(feature = "tokio")] +pub async fn devices() -> Result, SourceError> { + crate::utils::run_blocking(devices_blocking).await +} + +/// Lists the video capture devices on this machine. +/// +/// Enumeration queries the platform capture stack and can block briefly. +pub fn devices_blocking() -> Result, SourceError> { + backend::devices().map_err(SourceError::new) +} + +/// Configuration for a [`DeviceVideoSource`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeviceVideoSourceConfig { + /// Device to capture from. + #[cfg_attr(feature = "serde", serde(default))] + pub device: DeviceSelector, + /// Format requested from the device. + #[cfg_attr(feature = "serde", serde(default))] + pub format: DeviceFormatRequest, +} + +/// Pixel video source that captures frames from a video device, such as a +/// camera. +/// +/// Construction opens the device and negotiates the capture format, so +/// [`DeviceVideoSource::format`] is known before any frame is pumped. The +/// source never reaches the end of its stream — stop the pump that drives +/// it instead. +/// +/// Frames carry a monotonic `timestamp_us`. Each frame's `frame_metadata` +/// is pre-filled with the wall-clock capture time — the device's own +/// capture timestamp when the platform reports a valid one. +pub struct DeviceVideoSource { + config: DeviceVideoSourceConfig, + format: DeviceFormat, + session: backend::Session, +} + +impl DeviceVideoSource { + /// Creates the source. Device negotiation runs on the tokio blocking + /// pool. + /// + /// Requires a running tokio runtime. Use + /// [`DeviceVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: DeviceVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await + } + + /// Opens the configured device and negotiates the capture format. + /// + /// This can block until the device delivers its first frame, bounded by + /// a timeout. Construction fails on a missing device, a format request + /// the device cannot satisfy, or a platform without a capture backend. + pub fn new_blocking(config: DeviceVideoSourceConfig) -> Result { + let session = backend::Session::open(&config).map_err(SourceError::new)?; + let format = session.format(); + Ok(Self { config, format, session }) + } + + /// Returns the configuration the source was created with. + pub fn config(&self) -> &DeviceVideoSourceConfig { + &self.config + } + + /// Returns the negotiated capture format. + /// + /// The resolution matches what [`PixelVideoSource::resolution`] reports. + /// The frame format is what the device delivers before any conversion. + pub fn format(&self) -> DeviceFormat { + self.format + } +} + +impl PixelVideoSource for DeviceVideoSource { + fn resolution(&self) -> VideoResolution { + self.format.resolution + } + + // Backends bound every blocking wait so the stop token is observed + // within ~100ms even when the device stalls. + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + self.session.next_frame(stop).map_err(SourceError::new) + } +} + +impl fmt::Debug for DeviceVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DeviceVideoSource") + .field("config", &self.config) + .field("format", &self.format) + .finish_non_exhaustive() + } +} + +/// Error returned by device capture. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum DeviceVideoSourceError { + /// Device capture has no backend for this platform. + #[error("device capture is not supported on this platform")] + UnsupportedPlatform, + /// The requested device was not found. + #[error("capture device was not found")] + DeviceNotFound, + /// The configuration is invalid. + #[error("invalid device source configuration: {0}")] + InvalidConfig(&'static str), + /// The requested frame format is not supported by this platform's + /// backend. + #[error("device capture does not support frame format {0} on this platform")] + UnsupportedFrameFormat(DeviceFrameFormat), + /// The requested capture format is not available on the selected device. + #[error("capture format is not available on the device: {0}")] + UnsupportedFormat(DeviceFormat), + /// Timed out waiting for the device to deliver a frame. + #[error("timed out waiting for a frame from the capture device")] + FrameTimeout, + /// Captured frame bytes did not match the negotiated format. + #[error("invalid captured frame: {0}")] + InvalidFrame(&'static str), + /// Pixel conversion failed. + #[error("failed to convert captured frame to I420: {0}")] + Convert(&'static str), + /// Compressed frame decoding failed. + #[error("failed to decode compressed frame: {0}")] + Decode(String), + /// The platform capture stack reported an error. + #[error("capture device error: {0}")] + Backend(String), +} + +/// Builds the packet-trailer metadata that device frames are pre-filled +/// with. A metadata callback set on the pump takes precedence. +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn capture_frame_metadata( + capture_wall_time_us: u64, +) -> livekit::webrtc::video_frame::FrameMetadata { + livekit::webrtc::video_frame::FrameMetadata { + user_timestamp: Some(capture_wall_time_us), + frame_id: None, + user_data: None, + } +} + +/// Validates the platform-neutral parts of a configuration; `supported` +/// reports whether the backend can deliver a frame format. +#[cfg_attr(not(any(target_os = "macos", target_os = "linux")), allow(dead_code))] +fn validate_config( + config: &DeviceVideoSourceConfig, + supported: fn(DeviceFrameFormat) -> bool, +) -> Result<(), DeviceVideoSourceError> { + if let DeviceSelector::Id(id) = &config.device { + if id.is_empty() { + return Err(DeviceVideoSourceError::InvalidConfig("device id must be non-empty")); + } + } + + let validate_frame_format = |frame_format: DeviceFrameFormat| { + if !supported(frame_format) { + return Err(DeviceVideoSourceError::UnsupportedFrameFormat(frame_format)); + } + Ok(()) + }; + let validate_resolution = |resolution: VideoResolution| { + if resolution.width == 0 { + return Err(DeviceVideoSourceError::InvalidConfig("width must be non-zero")); + } + if resolution.height == 0 { + return Err(DeviceVideoSourceError::InvalidConfig("height must be non-zero")); + } + Ok(()) + }; + + match &config.format { + DeviceFormatRequest::Default => Ok(()), + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + validate_resolution(format.resolution)?; + if format.framerate_fps == 0 { + return Err(DeviceVideoSourceError::InvalidConfig( + "framerate_fps must be non-zero", + )); + } + validate_frame_format(format.frame_format) + } + DeviceFormatRequest::HighestFramerate { resolution, frame_format } => { + if let Some(resolution) = resolution { + validate_resolution(*resolution)?; + } + if let Some(frame_format) = frame_format { + validate_frame_format(*frame_format)?; + } + Ok(()) + } + DeviceFormatRequest::HighestResolution { framerate_fps, frame_format } => { + if matches!(framerate_fps, Some(0)) { + return Err(DeviceVideoSourceError::InvalidConfig( + "framerate_fps must be non-zero", + )); + } + if let Some(frame_format) = frame_format { + validate_frame_format(*frame_format)?; + } + Ok(()) + } + } +} + +/// Stub backend for platforms without device capture. +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +mod unsupported { + use livekit::webrtc::video_frame::BoxVideoFrame; + + use super::{DeviceFormat, DeviceInfo, DeviceVideoSourceConfig, DeviceVideoSourceError}; + use crate::pump::PumpStop; + + /// Uninhabited: [`Session::open`] always fails on this platform. + #[derive(Debug)] + pub(super) enum Session {} + + impl Session { + pub(super) fn open( + _config: &DeviceVideoSourceConfig, + ) -> Result { + Err(DeviceVideoSourceError::UnsupportedPlatform) + } + + pub(super) fn format(&self) -> DeviceFormat { + match *self {} + } + + pub(super) fn next_frame( + &mut self, + _stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + match *self {} + } + } + + pub(super) fn devices() -> Result, DeviceVideoSourceError> { + Err(DeviceVideoSourceError::UnsupportedPlatform) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use super::*; + + fn any_supported(_: DeviceFrameFormat) -> bool { + true + } + + #[test] + fn frame_format_parses_common_names() { + assert_eq!(DeviceFrameFormat::from_str("MJPEG"), Ok(DeviceFrameFormat::Mjpeg)); + assert_eq!(DeviceFrameFormat::from_str("mjpg"), Ok(DeviceFrameFormat::Mjpeg)); + assert_eq!(DeviceFrameFormat::from_str("grey"), Ok(DeviceFrameFormat::Grey)); + assert_eq!(DeviceFrameFormat::from_str("GREY"), Ok(DeviceFrameFormat::Grey)); + assert_eq!(DeviceFrameFormat::from_str("yuy2"), Ok(DeviceFrameFormat::Yuyv)); + } + + #[test] + fn frame_format_displays_canonical_names() { + assert_eq!(DeviceFrameFormat::Mjpeg.to_string(), "mjpeg"); + assert_eq!(DeviceFrameFormat::Grey.to_string(), "grey"); + } + + #[test] + fn validation_rejects_empty_device_id() { + let config = DeviceVideoSourceConfig { + device: DeviceSelector::Id(String::new()), + format: DeviceFormatRequest::Default, + }; + assert!(matches!( + validate_config(&config, any_supported), + Err(DeviceVideoSourceError::InvalidConfig(_)) + )); + } + + #[test] + fn validation_rejects_zero_format_components() { + let zero_width = DeviceVideoSourceConfig { + device: DeviceSelector::Default, + format: DeviceFormatRequest::Exact(DeviceFormat::new( + VideoResolution::new(0, 720), + 30, + DeviceFrameFormat::Yuyv, + )), + }; + assert!(matches!( + validate_config(&zero_width, any_supported), + Err(DeviceVideoSourceError::InvalidConfig(_)) + )); + + let zero_framerate = DeviceVideoSourceConfig { + device: DeviceSelector::Default, + format: DeviceFormatRequest::HighestResolution { + framerate_fps: Some(0), + frame_format: None, + }, + }; + assert!(matches!( + validate_config(&zero_framerate, any_supported), + Err(DeviceVideoSourceError::InvalidConfig(_)) + )); + } + + #[test] + fn validation_rejects_unsupported_frame_formats() { + let config = DeviceVideoSourceConfig { + device: DeviceSelector::Default, + format: DeviceFormatRequest::HighestFramerate { + resolution: None, + frame_format: Some(DeviceFrameFormat::Uyvy), + }, + }; + assert!(matches!( + validate_config(&config, |format| format != DeviceFrameFormat::Uyvy), + Err(DeviceVideoSourceError::UnsupportedFrameFormat(DeviceFrameFormat::Uyvy)) + )); + } + + #[test] + fn default_config_requests_default_device_and_format() { + let config = DeviceVideoSourceConfig::default(); + assert_eq!(config.device, DeviceSelector::Default); + assert_eq!(config.format, DeviceFormatRequest::Default); + } + + #[test] + fn device_info_selector_reopens_by_id() { + let info = DeviceInfo { + id: "camera-0".to_string(), + name: "Camera".to_string(), + model_id: None, + manufacturer: None, + formats: Vec::new(), + formats_complete: false, + }; + assert_eq!(info.selector(), DeviceSelector::Id("camera-0".to_string())); + } +} diff --git a/livekit-capture/src/sources/device/timestamp.rs b/livekit-capture/src/sources/device/timestamp.rs new file mode 100644 index 000000000..038e83ce1 --- /dev/null +++ b/livekit-capture/src/sources/device/timestamp.rs @@ -0,0 +1,65 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Capture-timestamp helpers shared by the device capture backends. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Maximum age a backend-reported capture timestamp can have, relative to +/// the wall-clock read time, before it is discarded as stale. +pub(super) const MAX_CAPTURE_TIMESTAMP_AGE_US: u64 = 5_000_000; + +/// Returns the current UNIX wall-clock time in microseconds. +pub(super) fn unix_time_us_now() -> Option { + let elapsed = SystemTime::now().duration_since(UNIX_EPOCH).ok()?; + u64::try_from(elapsed.as_micros()).ok() +} + +/// Converts a duration to whole microseconds, saturating at `i64::MAX`. +pub(super) fn elapsed_us(duration: Duration) -> i64 { + i64::try_from(duration.as_micros()).unwrap_or(i64::MAX) +} + +/// Validates a backend-reported capture timestamp against the wall-clock read +/// time: zero, future, and stale (older than +/// [`MAX_CAPTURE_TIMESTAMP_AGE_US`]) timestamps are rejected. +pub(super) fn validate_capture_timestamp_us( + capture_timestamp_us: u64, + read_wall_time_us: u64, +) -> Option { + if capture_timestamp_us == 0 || capture_timestamp_us > read_wall_time_us { + return None; + } + if read_wall_time_us - capture_timestamp_us > MAX_CAPTURE_TIMESTAMP_AGE_US { + return None; + } + Some(capture_timestamp_us) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validate_rejects_zero_future_and_stale_timestamps() { + let now = 10_000_000; + assert_eq!(validate_capture_timestamp_us(0, now), None); + assert_eq!(validate_capture_timestamp_us(now + 1, now), None); + assert_eq!( + validate_capture_timestamp_us(now - MAX_CAPTURE_TIMESTAMP_AGE_US - 1, now), + None + ); + assert_eq!(validate_capture_timestamp_us(now - 1, now), Some(now - 1)); + } +} diff --git a/livekit-capture/src/sources/device/v4l2.rs b/livekit-capture/src/sources/device/v4l2.rs new file mode 100644 index 000000000..e718fceb3 --- /dev/null +++ b/livekit-capture/src/sources/device/v4l2.rs @@ -0,0 +1,1136 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Linux device capture backend built on V4L2. +//! +//! This module is an implementation detail of [`super::DeviceVideoSource`]: +//! nothing V4L2-specific leaves it. Frames are converted to I420 on the CPU +//! (through libyuv, with an image-crate fallback for MJPEG streams that +//! libyuv rejects). + +use std::io; +use std::path::Path; +use std::time::{Duration, Instant}; + +use livekit::webrtc::video_frame::{BoxVideoFrame, I420Buffer, VideoFrame, VideoRotation}; +use v4l::{ + buffer::{Flags as V4lBufferFlags, Type as V4lBufferType}, + capability::Flags as V4lCapabilityFlags, + context, + format::{Format as V4lFormat, FourCC}, + frameinterval::FrameIntervalEnum, + framesize::FrameSizeEnum, + io::{mmap::Stream as MmapStream, traits::CaptureStream}, + video::{capture::Parameters as V4lCaptureParameters, Capture}, + Device, +}; + +use super::timestamp::{elapsed_us, unix_time_us_now, validate_capture_timestamp_us}; +use super::{ + capture_frame_metadata, DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, + DeviceSelector, DeviceVideoSourceConfig, DeviceVideoSourceError, +}; +use crate::{primitive::VideoResolution, pump::PumpStop}; + +/// How long the stream's own wait may block. Only the first frame read (which +/// starts the stream) can hit this; later reads are gated on a poll and never +/// wait inside the stream. A stream wait that times out cannot be retried, so +/// a timeout here is a hard [`DeviceVideoSourceError::FrameTimeout`]. +const FIRST_FRAME_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long one fd poll may block before the stop token is rechecked, in +/// milliseconds. +const STOP_CHECK_INTERVAL_MS: i32 = 100; + +/// Number of memory-mapped buffers shared with the driver. +const BUFFER_COUNT: u32 = 4; + +/// Returns whether the backend can convert this source frame format. +fn is_supported_source_format(frame_format: DeviceFrameFormat) -> bool { + matches!( + frame_format, + DeviceFrameFormat::Nv12 + | DeviceFrameFormat::Rgb24 + | DeviceFrameFormat::Bgr24 + | DeviceFrameFormat::Yuyv + | DeviceFrameFormat::Grey + | DeviceFrameFormat::Mjpeg + ) +} + +/// Default ordered source frame formats to try, most preferred first. +fn default_frame_formats() -> Vec { + vec![ + DeviceFrameFormat::Yuyv, + DeviceFrameFormat::Mjpeg, + DeviceFrameFormat::Grey, + DeviceFrameFormat::Rgb24, + DeviceFrameFormat::Nv12, + ] +} + +/// V4L2 capture session satisfying the backend contract. +pub(super) struct Session { + device: Device, + stream: MmapStream<'static>, + format: DeviceFormat, + // Driver-reported row stride in bytes (V4L2 `bytesperline`). + stride: u32, + started_at: Instant, + // Frame pulled while starting the stream, handed out first. + pending_frame: Option, +} + +impl Session { + /// Opens the device, negotiates the capture format, and starts the + /// stream by pulling its first frame. + pub(super) fn open(config: &DeviceVideoSourceConfig) -> Result { + super::validate_config(config, is_supported_source_format)?; + + let frame_formats = frame_formats_for_request(&config.format); + let device = open_device(&config.device)?; + let device_name = device + .query_caps() + .ok() + .map(|caps| caps.card) + .filter(|card| !card.is_empty()) + .unwrap_or_else(|| "unknown".to_string()); + let all_formats = enumerate_device_formats(&device)?; + let (format, stride) = + apply_format_request(&device, &config.format, &frame_formats, &all_formats)?; + let mut stream = + MmapStream::with_buffers(&device, V4lBufferType::VideoCapture, BUFFER_COUNT) + .map_err(backend_error)?; + stream.set_timeout(FIRST_FRAME_TIMEOUT); + + let mut session = Self { + device, + stream, + format, + stride, + started_at: Instant::now(), + pending_frame: None, + }; + // Pull the first frame during construction: it queues the stream's + // buffers and starts streaming, so every later wait can be + // poll-bounded to observe the stop token, and it proves the + // negotiated format actually delivers frames. + let first_frame = session.read_frame()?; + session.pending_frame = Some(first_frame); + log::info!("Opened device \"{}\": {} (converted to I420)", device_name, session.format,); + Ok(session) + } + + /// Returns the negotiated capture format. + pub(super) fn format(&self) -> DeviceFormat { + self.format + } + + /// Blocks until the next frame is available, returning `Ok(None)` once + /// the stop token fires. + pub(super) fn next_frame( + &mut self, + stop: &PumpStop, + ) -> Result, DeviceVideoSourceError> { + if let Some(frame) = self.pending_frame.take() { + return Ok(Some(frame)); + } + + // Bounded fd polls keep the stop token observed within + // STOP_CHECK_INTERVAL_MS even when the device stalls. The stream is + // only read once the fd signals, so the stream's own wait — which + // cannot be resumed after a timeout — never blocks here. + loop { + if stop.is_stopped() { + return Ok(None); + } + match self.device.handle().poll(libc::POLLIN, STOP_CHECK_INTERVAL_MS) { + Ok(0) => continue, + // Readable, or an error condition the stream read surfaces. + Ok(_) => break, + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, + Err(err) => return Err(backend_error(err)), + } + } + self.read_frame().map(Some) + } + + /// Dequeues one frame from the stream and converts it to I420. + fn read_frame(&mut self) -> Result { + let fallback_wall_time_us = unix_time_us_now().unwrap_or_default(); + let format = self.format; + let stride = self.stride; + let (buffer, metadata) = self.stream.next().map_err(|err| match err.kind() { + io::ErrorKind::TimedOut => DeviceVideoSourceError::FrameTimeout, + _ => backend_error(err), + })?; + let timestamp_us = elapsed_us(self.started_at.elapsed()); + let read_wall_time_us = unix_time_us_now().unwrap_or(fallback_wall_time_us); + let backend_capture_timestamp = + v4l_timestamp_to_wallclock(metadata.timestamp, v4l_timestamp_clock(metadata.flags)); + let capture_wall_time_us = select_capture_wall_time_us( + backend_capture_timestamp, + fallback_wall_time_us, + read_wall_time_us, + ); + + let width = format.resolution.width; + let height = format.resolution.height; + let mut i420 = I420Buffer::new(width, height); + let source = frame_bytes(buffer, metadata.bytesused); + convert_to_i420(format.frame_format, source, width, height, stride, &mut i420)?; + + Ok(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us, + frame_metadata: Some(capture_frame_metadata(capture_wall_time_us)), + buffer: Box::new(i420), + }) + } +} + +/// Lists Linux V4L2 capture devices. +pub(super) fn devices() -> Result, DeviceVideoSourceError> { + let devices = context::enum_devices() + .into_iter() + .filter_map(|node| { + let id = node.index().to_string(); + let fallback_name = + node.name().unwrap_or_else(|| node.path().to_string_lossy().into_owned()); + let mut name = fallback_name; + let mut model_id = None; + let mut manufacturer = None; + let mut formats = Vec::new(); + let mut formats_complete = false; + + if let Ok(device) = Device::with_path(node.path()) { + if let Ok(capabilities) = device.query_caps() { + if !capabilities.capabilities.contains(V4lCapabilityFlags::VIDEO_CAPTURE) { + return None; + } + if !capabilities.card.is_empty() { + name = capabilities.card; + } + model_id = Some(capabilities.bus).filter(|value| !value.is_empty()); + manufacturer = Some(capabilities.driver).filter(|value| !value.is_empty()); + } + + if let Ok(device_formats) = enumerate_device_formats(&device) { + formats = device_formats; + formats_complete = true; + } + } + + Some(DeviceInfo { id, name, model_id, manufacturer, formats, formats_complete }) + }) + .collect(); + + Ok(devices) +} + +fn open_device(selector: &DeviceSelector) -> Result { + match selector { + DeviceSelector::Default => Device::new(0).map_err(open_error), + DeviceSelector::Index(index) => Device::new(*index).map_err(open_error), + DeviceSelector::Id(id) => open_device_id(id), + } +} + +fn open_device_id(id: &str) -> Result { + if let Ok(index) = id.parse::() { + return Device::new(index).map_err(open_error); + } + + Device::with_path(Path::new(id)).map_err(open_error) +} + +fn open_error(error: io::Error) -> DeviceVideoSourceError { + if error.kind() == io::ErrorKind::NotFound { + DeviceVideoSourceError::DeviceNotFound + } else { + backend_error(error) + } +} + +fn backend_error(error: io::Error) -> DeviceVideoSourceError { + DeviceVideoSourceError::Backend(error.to_string()) +} + +/// Returns the ordered source frame formats to try for a request. +/// +/// The request's own frame format (already validated as supported) is tried +/// first; an explicit constraint on the highest-* requests pins the list to +/// that one format. +fn frame_formats_for_request(request: &DeviceFormatRequest) -> Vec { + let mut formats = match request { + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + ordered_formats_with_first(&default_frame_formats(), format.frame_format) + } + DeviceFormatRequest::HighestFramerate { frame_format: Some(frame_format), .. } + | DeviceFormatRequest::HighestResolution { frame_format: Some(frame_format), .. } => { + vec![*frame_format] + } + DeviceFormatRequest::Default + | DeviceFormatRequest::HighestFramerate { frame_format: None, .. } + | DeviceFormatRequest::HighestResolution { frame_format: None, .. } => { + default_frame_formats() + } + }; + formats.dedup(); + formats +} + +fn ordered_formats_with_first( + frame_formats: &[DeviceFrameFormat], + first: DeviceFrameFormat, +) -> Vec { + std::iter::once(first) + .chain(frame_formats.iter().copied().filter(|format| *format != first)) + .collect() +} + +fn apply_format_request( + device: &Device, + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + match request { + DeviceFormatRequest::Default + | DeviceFormatRequest::HighestFramerate { .. } + | DeviceFormatRequest::HighestResolution { .. } => { + let selected = select_format_for_request(request, frame_formats, all_formats)?; + set_device_format(device, selected) + } + DeviceFormatRequest::Exact(_) | DeviceFormatRequest::Closest(_) => { + apply_ordered_format_request(device, request, frame_formats, all_formats) + } + } +} + +/// Tries the request once per candidate source frame format, in preference +/// order, returning the first format the device accepts. +fn apply_ordered_format_request( + device: &Device, + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + let mut last_error = None; + for frame_format in frame_formats { + let request = format_request_with_frame_format(request, *frame_format); + let selected = match select_format_for_request(&request, &[*frame_format], all_formats) { + Ok(selected) => selected, + Err(error) => { + last_error = Some(error); + continue; + } + }; + + match set_device_format(device, selected) { + Ok(format) => return Ok(format), + Err(error) => last_error = Some(error), + } + } + + Err(last_error + .unwrap_or(DeviceVideoSourceError::InvalidConfig("no source frame formats to request"))) +} + +fn format_request_with_frame_format( + request: &DeviceFormatRequest, + frame_format: DeviceFrameFormat, +) -> DeviceFormatRequest { + match request { + DeviceFormatRequest::Exact(format) => DeviceFormatRequest::Exact(DeviceFormat::new( + format.resolution, + format.framerate_fps, + frame_format, + )), + DeviceFormatRequest::Closest(format) => DeviceFormatRequest::Closest(DeviceFormat::new( + format.resolution, + format.framerate_fps, + frame_format, + )), + DeviceFormatRequest::Default => DeviceFormatRequest::Default, + DeviceFormatRequest::HighestFramerate { resolution, .. } => { + DeviceFormatRequest::HighestFramerate { + resolution: *resolution, + frame_format: Some(frame_format), + } + } + DeviceFormatRequest::HighestResolution { framerate_fps, .. } => { + DeviceFormatRequest::HighestResolution { + framerate_fps: *framerate_fps, + frame_format: Some(frame_format), + } + } + } +} + +fn select_format_for_request( + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Result { + let selected = match request { + DeviceFormatRequest::Default => { + all_formats.iter().find(|format| frame_formats.contains(&format.frame_format)).copied() + } + DeviceFormatRequest::Exact(format) => { + if frame_formats.contains(&format.frame_format) { + Some(*format) + } else { + None + } + } + DeviceFormatRequest::Closest(format) => { + select_closest_format(*format, frame_formats, all_formats) + } + DeviceFormatRequest::HighestFramerate { .. } => { + select_highest_framerate_format(request, frame_formats, all_formats) + } + DeviceFormatRequest::HighestResolution { .. } => { + select_highest_resolution_format(request, frame_formats, all_formats) + } + }; + + selected.ok_or_else(|| match request { + DeviceFormatRequest::Exact(format) | DeviceFormatRequest::Closest(format) => { + DeviceVideoSourceError::UnsupportedFormat(*format) + } + _ => DeviceVideoSourceError::Backend( + "no device format satisfies the format request".to_string(), + ), + }) +} + +fn select_closest_format( + requested: DeviceFormat, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Option { + if !frame_formats.contains(&requested.frame_format) { + return None; + } + + let resolution = all_formats + .iter() + .copied() + .filter(|format| format.frame_format == requested.frame_format) + .min_by_key(|format| resolution_distance(format.resolution, requested.resolution))? + .resolution; + + let framerate_fps = all_formats + .iter() + .copied() + .filter(|format| { + format.frame_format == requested.frame_format && format.resolution == resolution + }) + .min_by_key(|format| format.framerate_fps.abs_diff(requested.framerate_fps))? + .framerate_fps; + + Some(DeviceFormat::new(resolution, framerate_fps, requested.frame_format)) +} + +fn select_highest_framerate_format( + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Option { + all_formats + .iter() + .copied() + .filter(|format| frame_formats.contains(&format.frame_format)) + .filter(|format| match request { + DeviceFormatRequest::HighestFramerate { resolution, frame_format } => { + resolution.map(|resolution| format.resolution == resolution).unwrap_or(true) + && frame_format + .map(|frame_format| format.frame_format == frame_format) + .unwrap_or(true) + } + _ => false, + }) + .max_by(|left, right| { + left.framerate_fps + .cmp(&right.framerate_fps) + .then_with(|| compare_resolution(left.resolution, right.resolution)) + .then_with(|| { + compare_format_preference(left.frame_format, right.frame_format, frame_formats) + }) + }) +} + +fn select_highest_resolution_format( + request: &DeviceFormatRequest, + frame_formats: &[DeviceFrameFormat], + all_formats: &[DeviceFormat], +) -> Option { + all_formats + .iter() + .copied() + .filter(|format| frame_formats.contains(&format.frame_format)) + .filter(|format| match request { + DeviceFormatRequest::HighestResolution { framerate_fps, frame_format } => { + framerate_fps + .map(|framerate_fps| format.framerate_fps == framerate_fps) + .unwrap_or(true) + && frame_format + .map(|frame_format| format.frame_format == frame_format) + .unwrap_or(true) + } + _ => false, + }) + .max_by(|left, right| { + compare_resolution(left.resolution, right.resolution) + .then_with(|| left.framerate_fps.cmp(&right.framerate_fps)) + .then_with(|| { + compare_format_preference(left.frame_format, right.frame_format, frame_formats) + }) + }) +} + +fn compare_resolution(left: VideoResolution, right: VideoResolution) -> std::cmp::Ordering { + frame_area(left) + .cmp(&frame_area(right)) + .then_with(|| left.width.cmp(&right.width)) + .then_with(|| left.height.cmp(&right.height)) +} + +fn resolution_distance(left: VideoResolution, right: VideoResolution) -> u64 { + let width = i64::from(left.width) - i64::from(right.width); + let height = i64::from(left.height) - i64::from(right.height); + width.unsigned_abs().pow(2) + height.unsigned_abs().pow(2) +} + +fn frame_area(resolution: VideoResolution) -> u64 { + u64::from(resolution.width) * u64::from(resolution.height) +} + +fn compare_format_preference( + left: DeviceFrameFormat, + right: DeviceFrameFormat, + frame_formats: &[DeviceFrameFormat], +) -> std::cmp::Ordering { + let left_index = frame_formats.iter().position(|format| *format == left).unwrap_or(usize::MAX); + let right_index = + frame_formats.iter().position(|format| *format == right).unwrap_or(usize::MAX); + right_index.cmp(&left_index) +} + +fn set_device_format( + device: &Device, + selected: DeviceFormat, +) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + let (current, _) = device_capture_format(device)?; + let format_changed = + current.resolution != selected.resolution || current.frame_format != selected.frame_format; + if format_changed { + device + .set_format(&V4lFormat::new( + selected.resolution.width, + selected.resolution.height, + fourcc_for_frame_format(selected.frame_format) + .ok_or(DeviceVideoSourceError::UnsupportedFrameFormat(selected.frame_format))?, + )) + .map_err(backend_error)?; + } + if format_changed || current.framerate_fps != selected.framerate_fps { + device + .set_params(&V4lCaptureParameters::with_fps(selected.framerate_fps)) + .map_err(backend_error)?; + } + + let (actual, stride) = device_capture_format(device)?; + if actual != selected { + return Err(DeviceVideoSourceError::Backend(format!( + "device rejected capture format: requested {selected}, got {actual}" + ))); + } + Ok((actual, stride)) +} + +/// Returns the device's current capture format and its row stride in bytes +/// (V4L2 `bytesperline`). +fn device_capture_format(device: &Device) -> Result<(DeviceFormat, u32), DeviceVideoSourceError> { + let format = device.format().map_err(backend_error)?; + let params = device.params().map_err(backend_error)?; + let framerate_fps = + framerate_from_fraction(params.interval.numerator, params.interval.denominator).ok_or( + DeviceVideoSourceError::Backend("device reports a zero frame interval".to_string()), + )?; + let capture_format = DeviceFormat::new( + VideoResolution::new(format.width, format.height), + framerate_fps, + frame_format_from_fourcc(format.fourcc).ok_or_else(|| { + DeviceVideoSourceError::Backend(format!("unsupported fourcc {}", format.fourcc)) + })?, + ); + Ok((capture_format, format.stride)) +} + +fn enumerate_device_formats(device: &Device) -> Result, DeviceVideoSourceError> { + let mut formats = Vec::new(); + let mut seen_fourccs = Vec::new(); + + for description in device.enum_formats().map_err(backend_error)? { + let fourcc = description.fourcc; + let Some(frame_format) = frame_format_from_fourcc(fourcc) else { + continue; + }; + if seen_fourccs.contains(&fourcc) { + continue; + } + seen_fourccs.push(fourcc); + let frame_sizes = device.enum_framesizes(fourcc).map_err(backend_error)?; + for resolution in frame_sizes.into_iter().flat_map(resolutions_from_frame_size) { + let intervals = device + .enum_frameintervals(fourcc, resolution.width, resolution.height) + .unwrap_or_default(); + for framerate_fps in intervals.into_iter().flat_map(framerates_from_interval) { + formats.push(DeviceFormat::new(resolution, framerate_fps, frame_format)); + } + } + } + + Ok(formats) +} + +fn fourcc_for_frame_format(frame_format: DeviceFrameFormat) -> Option { + match frame_format { + DeviceFrameFormat::Nv12 => Some(FourCC::new(b"NV12")), + DeviceFrameFormat::Rgb24 => Some(FourCC::new(b"RGB3")), + DeviceFrameFormat::Bgr24 => Some(FourCC::new(b"BGR3")), + DeviceFrameFormat::Yuyv => Some(FourCC::new(b"YUYV")), + DeviceFrameFormat::Grey => Some(FourCC::new(b"GREY")), + DeviceFrameFormat::Mjpeg => Some(FourCC::new(b"MJPG")), + DeviceFrameFormat::I420 | DeviceFrameFormat::Bgra | DeviceFrameFormat::Uyvy => None, + } +} + +fn frame_format_from_fourcc(fourcc: FourCC) -> Option { + match fourcc.str().ok()? { + "NV12" => Some(DeviceFrameFormat::Nv12), + "RGB3" => Some(DeviceFrameFormat::Rgb24), + "BGR3" => Some(DeviceFrameFormat::Bgr24), + "YUYV" | "YUY2" => Some(DeviceFrameFormat::Yuyv), + "GREY" => Some(DeviceFrameFormat::Grey), + "MJPG" | "JPEG" => Some(DeviceFrameFormat::Mjpeg), + _ => None, + } +} + +fn resolutions_from_frame_size(size: v4l::FrameSize) -> Vec { + match size.size { + FrameSizeEnum::Discrete(discrete) => { + vec![VideoResolution::new(discrete.width, discrete.height)] + } + FrameSizeEnum::Stepwise(stepwise) => { + let mut resolutions = Vec::new(); + push_stepwise_resolution( + &mut resolutions, + VideoResolution::new(stepwise.min_width, stepwise.min_height), + ); + push_stepwise_resolution( + &mut resolutions, + VideoResolution::new(stepwise.max_width, stepwise.max_height), + ); + resolutions + } + } +} + +fn push_stepwise_resolution(resolutions: &mut Vec, resolution: VideoResolution) { + if resolution.width != 0 && resolution.height != 0 && !resolutions.contains(&resolution) { + resolutions.push(resolution); + } +} + +fn framerates_from_interval(interval: v4l::FrameInterval) -> Vec { + match interval.interval { + FrameIntervalEnum::Discrete(fraction) => { + framerate_from_fraction(fraction.numerator, fraction.denominator).into_iter().collect() + } + FrameIntervalEnum::Stepwise(stepwise) => { + let mut framerates = Vec::new(); + for fraction in [stepwise.min, stepwise.max] { + if let Some(framerate) = + framerate_from_fraction(fraction.numerator, fraction.denominator) + { + if !framerates.contains(&framerate) { + framerates.push(framerate); + } + } + } + framerates + } + } +} + +/// Converts a V4L2 frame interval (seconds per frame) to frames per second. +/// +/// Non-integer rates (e.g. the NTSC interval 1001/30000 = 29.97fps) round to +/// the nearest whole rate, never below 1. +fn framerate_from_fraction(numerator: u32, denominator: u32) -> Option { + if numerator == 0 || denominator == 0 { + return None; + } + if denominator % numerator == 0 { + return Some(denominator / numerator); + } + let rounded = (u64::from(denominator) + u64::from(numerator) / 2) / u64::from(numerator); + Some(u32::try_from(rounded).unwrap_or(u32::MAX).max(1)) +} + +fn frame_bytes(buffer: &[u8], bytes_used: u32) -> &[u8] { + let bytes_used = usize::try_from(bytes_used).unwrap_or(buffer.len()).min(buffer.len()); + if bytes_used == 0 { + buffer + } else { + &buffer[..bytes_used] + } +} + +fn convert_to_i420( + source_format: DeviceFrameFormat, + source: &[u8], + width: u32, + height: u32, + source_stride: u32, + destination: &mut I420Buffer, +) -> Result<(), DeviceVideoSourceError> { + let (stride_y, stride_u, stride_v) = destination.strides(); + let (dst_y, dst_u, dst_v) = destination.data_mut(); + let width_i32 = i32_from_u32(width, "width exceeds supported range")?; + let height_i32 = i32_from_u32(height, "height exceeds supported range")?; + + let ret = match source_format { + DeviceFrameFormat::Yuyv => { + let stride = source_row_stride(source_stride, width as usize * 2); + validate_len(source, stride * height as usize, "YUYV frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_YUY2ToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Rgb24 => { + let stride = source_row_stride(source_stride, width as usize * 3); + validate_len(source, stride * height as usize, "RGB24 frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_RGB24ToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Bgr24 => { + let stride = source_row_stride(source_stride, width as usize * 3); + validate_len(source, stride * height as usize, "BGR24 frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_RAWToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Grey => { + let stride = source_row_stride(source_stride, width as usize); + validate_len(source, stride * height as usize, "GREY frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_I400ToI420( + source.as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Nv12 => { + // Single-planar V4L2 NV12: the interleaved chroma plane follows the + // luma plane at `stride * height` and shares the luma stride. + let stride = source_row_stride(source_stride, width as usize); + let y_size = stride * height as usize; + validate_len(source, y_size + y_size / 2, "NV12 frame is too short")?; + let stride_i32 = i32_from_usize(stride, "stride exceeds supported range")?; + // SAFETY: Source and destination slices are valid for the dimensions and strides. + unsafe { + yuv_sys::rs_NV12ToI420( + source.as_ptr(), + stride_i32, + source[y_size..].as_ptr(), + stride_i32, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + } + } + DeviceFrameFormat::Mjpeg => { + return convert_mjpeg_to_i420(source, width, height, destination); + } + DeviceFrameFormat::I420 | DeviceFrameFormat::Bgra | DeviceFrameFormat::Uyvy => { + return Err(DeviceVideoSourceError::UnsupportedFrameFormat(source_format)); + } + }; + + if ret == 0 { + Ok(()) + } else { + Err(DeviceVideoSourceError::Convert("libyuv conversion failed")) + } +} + +/// Returns the effective source row stride in bytes, falling back to the +/// packed width-derived stride when the driver reports `bytesperline` as zero +/// or smaller than one packed row. +fn source_row_stride(reported_stride: u32, packed_stride: usize) -> usize { + (reported_stride as usize).max(packed_stride) +} + +fn convert_mjpeg_to_i420( + source: &[u8], + width: u32, + height: u32, + destination: &mut I420Buffer, +) -> Result<(), DeviceVideoSourceError> { + let (stride_y, stride_u, stride_v) = destination.strides(); + let (dst_y, dst_u, dst_v) = destination.data_mut(); + let width_i32 = i32_from_u32(width, "width exceeds supported range")?; + let height_i32 = i32_from_u32(height, "height exceeds supported range")?; + + // SAFETY: Source and destination slices are valid for the dimensions and strides. + let ret = unsafe { + yuv_sys::rs_MJPGToI420( + source.as_ptr(), + source.len(), + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + width_i32, + height_i32, + ) + }; + if ret == 0 { + return Ok(()); + } + + let rgb = image::load_from_memory(source) + .map_err(|error| DeviceVideoSourceError::Decode(error.to_string()))? + .to_rgb8(); + if rgb.width() != width || rgb.height() != height { + return Err(DeviceVideoSourceError::InvalidFrame("decoded MJPEG dimensions changed")); + } + // SAFETY: Source and destination slices are valid for the dimensions and strides. + let ret = unsafe { + yuv_sys::rs_RGB24ToI420( + rgb.as_raw().as_ptr(), + width_i32 * 3, + dst_y.as_mut_ptr(), + stride_y as i32, + dst_u.as_mut_ptr(), + stride_u as i32, + dst_v.as_mut_ptr(), + stride_v as i32, + width_i32, + height_i32, + ) + }; + if ret == 0 { + Ok(()) + } else { + Err(DeviceVideoSourceError::Convert("RGB24 fallback conversion failed")) + } +} + +fn validate_len( + source: &[u8], + expected: usize, + label: &'static str, +) -> Result<(), DeviceVideoSourceError> { + if source.len() < expected { + return Err(DeviceVideoSourceError::InvalidFrame(label)); + } + Ok(()) +} + +/// Selects the wall-clock capture time for a frame: the validated +/// driver-reported timestamp when there is one, the read time otherwise. +fn select_capture_wall_time_us( + backend_capture_timestamp: Option, + fallback_wall_time_us: u64, + read_wall_time_us: u64, +) -> u64 { + backend_capture_timestamp + .and_then(|timestamp| u64::try_from(timestamp.as_micros()).ok()) + .and_then(|timestamp_us| validate_capture_timestamp_us(timestamp_us, read_wall_time_us)) + .unwrap_or(fallback_wall_time_us) +} + +fn i32_from_u32(value: u32, label: &'static str) -> Result { + i32::try_from(value).map_err(|_| DeviceVideoSourceError::InvalidFrame(label)) +} + +fn i32_from_usize(value: usize, label: &'static str) -> Result { + i32::try_from(value).map_err(|_| DeviceVideoSourceError::InvalidFrame(label)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum V4lTimestampClock { + Unknown, + Monotonic, + Copy, + Unsupported, +} + +fn v4l_timestamp_clock(flags: V4lBufferFlags) -> V4lTimestampClock { + let timestamp_type = flags.bits() & V4lBufferFlags::TIMESTAMP_MASK.bits(); + if timestamp_type == V4lBufferFlags::TIMESTAMP_MONOTONIC.bits() { + V4lTimestampClock::Monotonic + } else if timestamp_type == V4lBufferFlags::TIMESTAMP_COPY.bits() { + V4lTimestampClock::Copy + } else if timestamp_type == V4lBufferFlags::TIMESTAMP_UNKNOWN.bits() { + V4lTimestampClock::Unknown + } else { + V4lTimestampClock::Unsupported + } +} + +fn v4l_timestamp_to_wallclock( + timestamp: v4l::Timestamp, + clock: V4lTimestampClock, +) -> Option { + let frame_timestamp = Duration::from(timestamp); + if frame_timestamp.is_zero() { + return None; + } + + let monotonic_now = clock_time(libc::CLOCK_MONOTONIC)?; + let wall_now = clock_time(libc::CLOCK_REALTIME)?; + timestamp_to_wallclock(frame_timestamp, clock, monotonic_now, wall_now) +} + +fn timestamp_to_wallclock( + frame_timestamp: Duration, + clock: V4lTimestampClock, + monotonic_now: Duration, + wall_now: Duration, +) -> Option { + if frame_timestamp.is_zero() { + return None; + } + + match clock { + V4lTimestampClock::Monotonic => { + monotonic_timestamp_to_wallclock(frame_timestamp, monotonic_now, wall_now) + } + V4lTimestampClock::Unknown => { + monotonic_timestamp_to_wallclock(frame_timestamp, monotonic_now, wall_now) + .or(Some(frame_timestamp)) + } + V4lTimestampClock::Copy | V4lTimestampClock::Unsupported => None, + } +} + +fn monotonic_timestamp_to_wallclock( + frame_timestamp: Duration, + monotonic_now: Duration, + wall_now: Duration, +) -> Option { + let frame_age = monotonic_now.checked_sub(frame_timestamp)?; + wall_now.checked_sub(frame_age) +} + +fn clock_time(clock_id: libc::clockid_t) -> Option { + let mut time = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `time` is a valid out pointer and `clock_id` is supplied by libc constants. + let ret = unsafe { libc::clock_gettime(clock_id, &mut time) }; + if ret != 0 || time.tv_sec < 0 || time.tv_nsec < 0 { + return None; + } + + Some(Duration::new(time.tv_sec as u64, time.tv_nsec as u32)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sources::device::timestamp::MAX_CAPTURE_TIMESTAMP_AGE_US; + + #[test] + fn source_formats_exclude_unconvertible_ones() { + assert!(!is_supported_source_format(DeviceFrameFormat::I420)); + assert!(!is_supported_source_format(DeviceFrameFormat::Bgra)); + assert!(!is_supported_source_format(DeviceFrameFormat::Uyvy)); + assert!(is_supported_source_format(DeviceFrameFormat::Yuyv)); + } + + #[test] + fn frame_formats_for_request_prefers_the_requested_format() { + let request = DeviceFormatRequest::Exact(DeviceFormat::new( + VideoResolution::new(1280, 720), + 30, + DeviceFrameFormat::Mjpeg, + )); + let formats = frame_formats_for_request(&request); + assert_eq!(formats.first(), Some(&DeviceFrameFormat::Mjpeg)); + assert_eq!(formats.len(), default_frame_formats().len()); + } + + #[test] + fn frame_formats_for_request_pins_explicit_constraints() { + let request = DeviceFormatRequest::HighestFramerate { + resolution: None, + frame_format: Some(DeviceFrameFormat::Grey), + }; + assert_eq!(frame_formats_for_request(&request), vec![DeviceFrameFormat::Grey]); + } + + #[test] + fn ignores_stream_relative_capture_timestamp() { + // A small timestamp (relative to stream start rather than a clock) + // fails wall-clock validation and falls back to the read time. + let selected = + select_capture_wall_time_us(Some(Duration::from_micros(5)), 9_000_000, 10_000_000); + assert_eq!(selected, 9_000_000); + } + + #[test] + fn accepts_recent_backend_capture_timestamp() { + let selected = select_capture_wall_time_us( + Some(Duration::from_micros(9_999_000)), + 9_000_000, + 10_000_000, + ); + assert_eq!(selected, 9_999_000); + } + + #[test] + fn ignores_backend_capture_timestamp_older_than_max_age() { + let read_wall_time_us = 10_000_000 + MAX_CAPTURE_TIMESTAMP_AGE_US; + let selected = select_capture_wall_time_us( + Some(Duration::from_micros(10_000_000 - 1)), + 9_000_000, + read_wall_time_us, + ); + assert_eq!(selected, 9_000_000); + } + + #[test] + fn converts_monotonic_v4l_timestamp_to_wallclock() { + let converted = timestamp_to_wallclock( + Duration::from_secs(90), + V4lTimestampClock::Monotonic, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, Some(Duration::from_secs(990))); + } + + #[test] + fn infers_unknown_v4l_timestamp_clock() { + // Convertible as monotonic: treated as monotonic. + let converted = timestamp_to_wallclock( + Duration::from_secs(90), + V4lTimestampClock::Unknown, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, Some(Duration::from_secs(990))); + + // Ahead of the monotonic clock: passed through as-is. + let converted = timestamp_to_wallclock( + Duration::from_secs(500), + V4lTimestampClock::Unknown, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, Some(Duration::from_secs(500))); + } + + #[test] + fn rejects_copied_and_unsupported_v4l_timestamps() { + for clock in [V4lTimestampClock::Copy, V4lTimestampClock::Unsupported] { + let converted = timestamp_to_wallclock( + Duration::from_secs(90), + clock, + Duration::from_secs(100), + Duration::from_secs(1_000), + ); + assert_eq!(converted, None); + } + } + + #[test] + fn framerate_from_fraction_rounds_fractional_intervals() { + assert_eq!(framerate_from_fraction(1, 30), Some(30)); + assert_eq!(framerate_from_fraction(1001, 30000), Some(30)); + assert_eq!(framerate_from_fraction(1001, 60000), Some(60)); + } + + #[test] + fn framerate_from_fraction_rejects_zero_terms() { + assert_eq!(framerate_from_fraction(0, 30), None); + assert_eq!(framerate_from_fraction(30, 0), None); + } +} diff --git a/livekit-capture/src/sources/gstreamer.rs b/livekit-capture/src/sources/gstreamer.rs new file mode 100644 index 000000000..92ba19b18 --- /dev/null +++ b/livekit-capture/src/sources/gstreamer.rs @@ -0,0 +1,1058 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Encoded video capture from a GStreamer pipeline. +//! +//! [`GStreamerVideoSource`] owns a pipeline that ends in an appsink and +//! yields the pipeline's encoded output as access units. + +use ::gstreamer as gst; +use ::gstreamer_app as gst_app; +use bytes::Bytes; +use gst::glib; +use gst::prelude::*; +use thiserror::Error; + +use crate::{ + encoded::{ + h26x::{access_unit_from_annex_b, access_unit_from_h264_avc, H26xParseError}, + EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, OwnedEncodedAccessUnit, + }, + error::SourceError, + primitive::VideoResolution, + pump::PumpStop, +}; +use livekit::webrtc::video_source::EncodedRateControl; + +/// Encoded sample format expected from a GStreamer appsink. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GStreamerSampleFormat { + /// H.264 Annex-B access units, usually from `h264parse` with byte-stream caps. + H264AnnexB, + /// H.264 access units with AVC length-prefixed NAL units. + H264Avc { + /// Length-prefix size in bytes. + nal_length_size: u8, + }, + /// H.265 Annex-B access units, usually from `h265parse` with byte-stream caps. + H265AnnexB, + /// One already-delimited encoded access unit per appsink sample. + AccessUnit { + /// Codec carried by each appsink sample. + codec: EncodedVideoCodec, + }, +} + +impl GStreamerSampleFormat { + /// Returns the encoded codec carried by this sample format. + fn codec(self) -> EncodedVideoCodec { + match self { + Self::H264AnnexB => EncodedVideoCodec::H264, + Self::H264Avc { .. } => EncodedVideoCodec::H264, + Self::H265AnnexB => EncodedVideoCodec::H265, + Self::AccessUnit { codec } => codec, + } + } +} + +/// Configuration for a GStreamer encoded video source. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct GStreamerVideoSourceConfig { + /// GStreamer launch description for the encoded producer pipeline. + /// + /// The pipeline must contain `appsink name=lk_appsink`, or leave exactly + /// one encoded video source pad unlinked. The source then attaches an + /// appsink to that pad. + pub pipeline: String, + + /// Codec expected from the pipeline. When omitted, the codec is + /// inferred from the pipeline caps. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub codec: Option, + + /// Encoded frame resolution. + /// + /// When omitted, the resolution is discovered from the first sample, so + /// construction waits for the pipeline to produce data. When set, + /// construction returns without waiting, and the first sample is + /// verified against this value. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub resolution: Option, + + /// Forwards WebRTC rate-control targets to an encoder element's bitrate + /// property. Without this, the pipeline encodes at a fixed bitrate. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub rate_control: Option, +} + +/// Binding from WebRTC rate-control targets to a GStreamer encoder property. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct GStreamerRateControlConfig { + /// Name of the encoder element in the pipeline (for example + /// `lk_encoder`). + pub element: String, + + /// Bitrate property to set on the element (for example `bitrate` for + /// x264enc, or `target-bitrate` for vp8enc/vp9enc). + pub property: String, + + /// Unit the property expects. + pub unit: GStreamerBitrateUnit, +} + +/// Bitrate unit used by a GStreamer encoder property. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum GStreamerBitrateUnit { + /// The encoder property expects bits per second. + #[cfg_attr(feature = "serde", serde(rename = "bps"))] + BitsPerSecond, + /// The encoder property expects kilobits per second. + #[cfg_attr(feature = "serde", serde(rename = "kbps"))] + KilobitsPerSecond, +} + +impl GStreamerBitrateUnit { + fn property_value(self, target_bitrate_bps: u64) -> u64 { + match self { + Self::BitsPerSecond => target_bitrate_bps, + Self::KilobitsPerSecond => target_bitrate_bps.div_ceil(1000), + } + } +} + +/// GStreamer encoder bitrate control used by [`GStreamerVideoSource`]. +#[derive(Debug, Clone)] +struct GStreamerEncoderRateControl { + encoder: gst::Element, + bitrate_property: String, + bitrate_unit: GStreamerBitrateUnit, + last_target_bitrate_bps: Option, +} + +impl GStreamerEncoderRateControl { + /// Creates bitrate control for a GStreamer encoder element. + fn new( + encoder: gst::Element, + bitrate_property: &str, + bitrate_unit: GStreamerBitrateUnit, + ) -> Self { + Self { + encoder, + bitrate_property: bitrate_property.to_owned(), + bitrate_unit, + last_target_bitrate_bps: None, + } + } + + fn update(&mut self, rate_control: EncodedRateControl) { + if self.last_target_bitrate_bps == Some(rate_control.target_bitrate_bps) { + return; + } + + let property_value = self.bitrate_unit.property_value(rate_control.target_bitrate_bps); + if set_integer_property(&self.encoder, &self.bitrate_property, property_value) { + self.last_target_bitrate_bps = Some(rate_control.target_bitrate_bps); + log::debug!( + "updated GStreamer encoder '{}' {}={} for WebRTC target {} bps at {:.2} fps", + self.encoder.name(), + self.bitrate_property, + property_value, + rate_control.target_bitrate_bps, + rate_control.framerate_fps, + ); + } + } +} + +/// How long one appsink wait may block before the stop token is rechecked. +const SAMPLE_WAIT: gst::ClockTime = gst::ClockTime::from_mseconds(100); + +/// How long stream discovery waits for the pipeline's first sample. +const DISCOVERY_TIMEOUT: gst::ClockTime = gst::ClockTime::from_seconds(5); + +/// Fallback frame interval when neither caps nor buffers carry timing. +const DEFAULT_FRAME_INTERVAL_US: i64 = 1_000_000 / 30; + +/// Encoded source that owns a GStreamer pipeline that ends in an appsink. +#[derive(Debug)] +pub struct GStreamerVideoSource { + pipeline: gst::Pipeline, + bus: gst::Bus, + appsink: gst_app::AppSink, + sample_format: GStreamerSampleFormat, + resolution: VideoResolution, + frame_interval_us: i64, + next_fallback_timestamp_us: i64, + rate_control: Option, + // Caps the stream was validated against; a pointer change on a later + // sample triggers revalidation. + negotiated_caps: Option, + // Sample pulled during stream discovery, handed out first. + pending_sample: Option, +} + +impl GStreamerVideoSource { + /// Creates the source. Construction and stream discovery run on the + /// tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`GStreamerVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: GStreamerVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await + } + + /// Builds and starts the GStreamer pipeline from the configuration. + /// + /// The pipeline starts to play immediately and returns to `Null` when + /// the source is dropped. Construction fails on an invalid launch + /// description, a missing appsink or encoded pad, a missing + /// rate-control element, or a pipeline that does not start. + /// + /// When the configuration declares no resolution, this blocks until the + /// first sample arrives (bounded by a timeout) to read the stream + /// settings. + pub fn new_blocking(config: GStreamerVideoSourceConfig) -> Result { + gst::init().map_err(|err| { + SourceError::new(GStreamerVideoSourceError::Pipeline(format!( + "failed to initialize GStreamer: {err}" + ))) + })?; + + let pipeline = gst::parse::launch(&config.pipeline) + .map_err(|err| { + SourceError::new(GStreamerVideoSourceError::Pipeline(format!( + "failed to create pipeline: {err}" + ))) + })? + .downcast::() + .map_err(|_| SourceError::new(GStreamerVideoSourceError::NotAPipeline))?; + + let (appsink, sample_format) = ensure_encoded_appsink(&pipeline, config.codec) + .map_err(|err| SourceError::new(GStreamerVideoSourceError::Layout(err)))?; + + let rate_control = config + .rate_control + .map(|binding| -> Result { + let encoder = pipeline.by_name(&binding.element).ok_or_else(|| { + GStreamerVideoSourceError::MissingRateControlElement(binding.element.clone()) + })?; + Ok(GStreamerEncoderRateControl::new(encoder, &binding.property, binding.unit)) + }) + .transpose() + .map_err(SourceError::new)?; + + let bus = pipeline.bus().ok_or_else(|| { + SourceError::new(GStreamerVideoSourceError::Pipeline( + "pipeline has no message bus".to_owned(), + )) + })?; + + pipeline.set_state(gst::State::Playing).map_err(|err| { + SourceError::new(GStreamerVideoSourceError::Pipeline(format!( + "failed to start pipeline: {err}" + ))) + })?; + + let mut source = Self { + pipeline, + bus, + appsink, + sample_format, + resolution: config.resolution.unwrap_or_default(), + frame_interval_us: DEFAULT_FRAME_INTERVAL_US, + next_fallback_timestamp_us: 0, + rate_control, + negotiated_caps: None, + pending_sample: None, + }; + + // Without a declared resolution, discover the stream settings from + // the first sample's negotiated caps; the sample is buffered so no + // keyframe is lost. A declared resolution skips the wait and is + // verified lazily against the first sample instead. + if config.resolution.is_none() { + let sample = source.wait_first_sample().map_err(SourceError::new)?; + let caps = sample + .caps() + .ok_or(GStreamerVideoSourceError::MissingResolutionCaps) + .map_err(SourceError::new)?; + source.resolution = resolution_from_caps(caps) + .ok_or(GStreamerVideoSourceError::MissingResolutionCaps) + .map_err(SourceError::new)?; + if let Some(frame_interval_us) = frame_interval_from_caps(caps) { + source.frame_interval_us = frame_interval_us; + } + source.negotiated_caps = Some(caps.to_owned()); + source.pending_sample = Some(sample); + } + + log::info!( + "GStreamer pipeline ready: {:?} {} ({} resolution)", + source.sample_format.codec(), + source.resolution, + if config.resolution.is_none() { "discovered" } else { "declared" }, + ); + Ok(source) + } + + /// Blocks until the pipeline produces its first sample, a bus error + /// arrives, or the discovery timeout expires. + fn wait_first_sample(&self) -> Result { + let deadline = + std::time::Instant::now() + std::time::Duration::from_secs(DISCOVERY_TIMEOUT.seconds()); + loop { + self.check_bus()?; + if let Some(sample) = self.appsink.try_pull_sample(SAMPLE_WAIT) { + return Ok(sample); + } + if self.appsink.is_eos() { + return Err(GStreamerVideoSourceError::EndedBeforeFirstSample); + } + if std::time::Instant::now() >= deadline { + return Err(GStreamerVideoSourceError::DiscoveryTimeout); + } + } + } + + /// Returns the GStreamer pipeline. + pub fn pipeline(&self) -> &gst::Pipeline { + &self.pipeline + } + + /// Returns a pending pipeline bus error, if any. + fn check_bus(&self) -> Result<(), GStreamerVideoSourceError> { + while let Some(message) = self.bus.pop_filtered(&[gst::MessageType::Error]) { + if let gst::MessageView::Error(error) = message.view() { + return Err(GStreamerVideoSourceError::Pipeline(format!( + "{} ({})", + error.error(), + error.debug().map(|s| s.to_string()).unwrap_or_default(), + ))); + } + } + Ok(()) + } + + /// Validates a sample's caps against the established stream settings. + fn check_caps(&mut self, sample: &gst::Sample) -> Result<(), GStreamerVideoSourceError> { + let Some(caps) = sample.caps() else { + return Ok(()); + }; + // Caps are immutable and refcounted, so an unchanged stream passes + // with a pointer comparison. On a caps change, the resolution and + // codec must match: live stream reconfiguration would require + // republishing the track, which is not supported yet. + if let Some(seen) = &self.negotiated_caps { + if seen.as_ptr() == caps.as_ptr() { + return Ok(()); + } + } + let had_baseline = self.negotiated_caps.is_some(); + + if let Some(structure) = caps.structure(0) { + if let Some(codec) = codec_from_caps_name(structure.name()) { + if codec != self.sample_format.codec() { + return Err(GStreamerVideoSourceError::Renegotiated { + from: format!("{:?}", self.sample_format.codec()), + to: format!("{codec:?}"), + }); + } + } + } + if let Some(resolution) = resolution_from_caps(caps) { + if resolution != self.resolution { + return Err(if had_baseline { + GStreamerVideoSourceError::Renegotiated { + from: self.resolution.to_string(), + to: resolution.to_string(), + } + } else { + GStreamerVideoSourceError::ResolutionMismatch { + configured: self.resolution, + actual: resolution, + } + }); + } + } + if let Some(frame_interval_us) = frame_interval_from_caps(caps) { + self.frame_interval_us = frame_interval_us; + } + + self.negotiated_caps = Some(caps.to_owned()); + Ok(()) + } + + fn process_sample( + &mut self, + sample: &gst::Sample, + ) -> Result { + self.check_caps(sample)?; + self.access_unit_from_sample(sample) + } + + fn access_unit_from_sample( + &mut self, + sample: &gst::Sample, + ) -> Result { + let buffer = sample.buffer().ok_or(GStreamerVideoSourceError::MissingBuffer)?; + let timestamp_us = self.timestamp_us(buffer); + let frame_type = if buffer.flags().contains(gst::BufferFlags::DELTA_UNIT) { + EncodedFrameType::Delta + } else { + EncodedFrameType::Key + }; + + let map = buffer + .map_readable() + .map_err(|err| GStreamerVideoSourceError::MapReadable(err.to_string()))?; + let payload = map.as_ref(); + access_unit_from_sample_payload( + self.sample_format, + payload, + timestamp_us, + frame_type, + self.resolution, + ) + .map_err(GStreamerVideoSourceError::Parse) + } + + fn timestamp_us(&mut self, buffer: &gst::BufferRef) -> i64 { + if let Some(timestamp) = buffer.pts().or_else(|| buffer.dts()) { + let timestamp_us = clock_time_to_timestamp_us(0, timestamp); + self.next_fallback_timestamp_us = timestamp_us.saturating_add(self.frame_interval_us); + return timestamp_us; + } + + let timestamp_us = self.next_fallback_timestamp_us; + self.next_fallback_timestamp_us = + self.next_fallback_timestamp_us.saturating_add(self.frame_interval_us); + timestamp_us + } +} + +impl Drop for GStreamerVideoSource { + fn drop(&mut self) { + // Returning the pipeline to `Null` releases its resources; GStreamer + // does not stop a running pipeline on the last unref. + let _ = self.pipeline.set_state(gst::State::Null); + } +} + +impl EncodedVideoSource for GStreamerVideoSource { + fn resolution(&self) -> VideoResolution { + self.resolution + } + + fn codec(&self) -> EncodedVideoCodec { + self.sample_format.codec() + } + + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError> { + if let Some(sample) = self.pending_sample.take() { + return self.process_sample(&sample).map(Some).map_err(SourceError::new); + } + + // Bounded waits keep the stop token observed within `SAMPLE_WAIT` + // even while the pipeline produces nothing. + loop { + if stop.is_stopped() { + return Ok(None); + } + self.check_bus().map_err(SourceError::new)?; + + match self.appsink.try_pull_sample(SAMPLE_WAIT) { + Some(sample) => { + return self.process_sample(&sample).map(Some).map_err(SourceError::new); + } + None if self.appsink.is_eos() => return Ok(None), + None => {} + } + } + } + + fn request_keyframe(&mut self) { + // The `GstForceKeyUnit` custom upstream event is understood by every + // GStreamer video encoder (it is what gst-video's force-key-unit + // helper builds), so downstream PLI/FIR reaches the producer. + let structure = + gst::Structure::builder("GstForceKeyUnit").field("all-headers", true).build(); + let _ = self.appsink.send_event(gst::event::CustomUpstream::new(structure)); + } + + fn update_rate_control(&mut self, rate_control: EncodedRateControl) { + if let Some(control) = &mut self.rate_control { + control.update(rate_control); + } + } +} + +fn set_integer_property(element: &gst::Element, property: &str, value: u64) -> bool { + let Some(pspec) = element.find_property(property) else { + log::warn!("GStreamer encoder '{}' has no '{property}' property", element.name()); + return false; + }; + + let flags = pspec.flags(); + if !flags.contains(glib::ParamFlags::WRITABLE) + || flags.contains(glib::ParamFlags::CONSTRUCT_ONLY) + { + log::warn!("GStreamer encoder '{}' property '{property}' is not writable", element.name()); + return false; + } + + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property( + property, + value.clamp(pspec.minimum() as u64, pspec.maximum() as u64) as u32, + ); + return true; + } + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property( + property, + clamp_to_i64(value, pspec.minimum() as i64, pspec.maximum() as i64) as i32, + ); + return true; + } + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property(property, value.clamp(pspec.minimum(), pspec.maximum())); + return true; + } + if let Some(pspec) = pspec.downcast_ref::() { + element.set_property(property, clamp_to_i64(value, pspec.minimum(), pspec.maximum())); + return true; + } + + log::warn!( + "GStreamer encoder '{}' property '{property}' has unsupported type '{}'", + element.name(), + pspec.value_type() + ); + false +} + +fn clamp_to_i64(value: u64, minimum: i64, maximum: i64) -> i64 { + let value = value.min(i64::MAX as u64) as i64; + value.clamp(minimum, maximum) +} + +/// Error returned by GStreamer appsink encoded sources. +#[derive(Debug, Error)] +pub enum GStreamerVideoSourceError { + /// The launch description did not produce a pipeline. + #[error("GStreamer description did not create a pipeline")] + NotAPipeline, + /// The pipeline produced no data during stream discovery. + #[error( + "pipeline produced no data during stream discovery; declare `resolution` in the \ + configuration to skip discovery, or check that the pipeline produces encoded video" + )] + DiscoveryTimeout, + /// The stream ended before producing a sample. + #[error("pipeline reached end of stream before producing a sample")] + EndedBeforeFirstSample, + /// Negotiated caps carry no resolution to discover. + #[error("negotiated caps declare no resolution; declare `resolution` in the configuration")] + MissingResolutionCaps, + /// The pipeline produces a different resolution than configured. + #[error("pipeline produces {actual}, but the configuration declares {configured}")] + ResolutionMismatch { + /// Resolution declared in the configuration. + configured: VideoResolution, + /// Resolution the pipeline negotiated. + actual: VideoResolution, + }, + /// Stream settings changed mid-stream. + #[error( + "pipeline renegotiated {from} to {to}; changing stream settings requires republishing \ + the track, which is not supported yet" + )] + Renegotiated { + /// Established stream setting. + from: String, + /// Newly negotiated stream setting. + to: String, + }, + /// The rate-control element is missing from the pipeline. + #[error("pipeline has no element named '{0}' for rate control")] + MissingRateControlElement(String), + /// The pipeline could not be built or started, or errored at runtime. + #[error("GStreamer pipeline error: {0}")] + Pipeline(String), + /// The pipeline layout cannot feed an encoded appsink. + #[error(transparent)] + Layout(#[from] GStreamerPipelineError), + /// The sample did not contain an encoded buffer. + #[error("GStreamer sample did not contain a buffer")] + MissingBuffer, + /// The sample buffer could not be mapped for reading. + #[error("failed to map GStreamer buffer for reading: {0}")] + MapReadable(String), + /// Access-unit parsing failed. + #[error(transparent)] + Parse(H26xParseError), +} + +fn access_unit_from_sample_payload( + sample_format: GStreamerSampleFormat, + payload: &[u8], + timestamp_us: i64, + frame_type: EncodedFrameType, + resolution: VideoResolution, +) -> Result { + match sample_format { + GStreamerSampleFormat::H264AnnexB => access_unit_from_annex_b( + EncodedVideoCodec::H264, + Bytes::copy_from_slice(payload), + timestamp_us, + resolution, + ), + GStreamerSampleFormat::H264Avc { nal_length_size } => { + access_unit_from_h264_avc(payload, nal_length_size, timestamp_us, resolution) + } + GStreamerSampleFormat::H265AnnexB => access_unit_from_annex_b( + EncodedVideoCodec::H265, + Bytes::copy_from_slice(payload), + timestamp_us, + resolution, + ), + GStreamerSampleFormat::AccessUnit { codec } => { + if payload.is_empty() { + return Err(H26xParseError::EmptyPayload); + } + + Ok(OwnedEncodedAccessUnit::new( + codec, + Bytes::copy_from_slice(payload), + timestamp_us, + frame_type, + resolution, + )) + } + } +} + +/// Reads the frame resolution from negotiated caps, when declared. +fn resolution_from_caps(caps: &gst::CapsRef) -> Option { + let structure = caps.structure(0)?; + let width = structure.get::("width").ok()?; + let height = structure.get::("height").ok()?; + (width > 0 && height > 0).then(|| VideoResolution::new(width as u32, height as u32)) +} + +/// Derives the fallback frame interval from the caps framerate, when +/// declared and non-zero. +fn frame_interval_from_caps(caps: &gst::CapsRef) -> Option { + let framerate = caps.structure(0)?.get::("framerate").ok()?; + let (numer, denom) = (i64::from(framerate.numer()), i64::from(framerate.denom())); + (numer > 0 && denom > 0).then(|| 1_000_000 * denom / numer) +} + +fn clock_time_to_timestamp_us(start_timestamp_us: i64, timestamp: gst::ClockTime) -> i64 { + let timestamp_us = timestamp.useconds().min(i64::MAX as u64) as i64; + start_timestamp_us.saturating_add(timestamp_us) +} + +/// Name of the appsink element the pipeline helpers look up or create. +pub const ENCODED_APPSINK_NAME: &str = "lk_appsink"; + +/// Error returned by the GStreamer pipeline helpers. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum GStreamerPipelineError { + /// The requested codec does not match what the pipeline advertises. + #[error( + "GStreamer codec mismatch: requested {requested:?}, but {location} advertises {advertised:?}" + )] + CodecMismatch { + /// Codec requested by the caller. + requested: EncodedVideoCodec, + /// Codec advertised by the pipeline. + advertised: EncodedVideoCodec, + /// Pipeline location that advertised the codec. + location: String, + }, + /// The pipeline has no usable appsink and no unlinked encoded pad. + #[error( + "GStreamer pipeline must include `appsink name={ENCODED_APPSINK_NAME}` or leave one \ + encoded video source pad unlinked" + )] + MissingAppSink, + /// The named element exists but is not an appsink. + #[error("GStreamer element {ENCODED_APPSINK_NAME} is not an appsink")] + NotAnAppSink, + /// Pad caps advertise no supported encoded video codec. + #[error("unlinked GStreamer pad '{0}' does not advertise supported encoded video caps")] + UnsupportedPadCaps(String), + /// Caps advertise a stream layout the encoded sources cannot consume. + #[error("unsupported GStreamer caps: {0}")] + UnsupportedCaps(String), + /// Element creation or linking failed. + #[error("{0}")] + Pipeline(String), +} + +/// Returns the appsink caps for a codec as a launch-string fragment. +pub fn encoded_caps_string(codec: EncodedVideoCodec) -> &'static str { + match codec { + EncodedVideoCodec::H264 => "video/x-h264,stream-format=byte-stream,alignment=au", + EncodedVideoCodec::H265 => "video/x-h265,stream-format=byte-stream,alignment=au", + EncodedVideoCodec::VP8 => "video/x-vp8", + EncodedVideoCodec::VP9 => "video/x-vp9,profile=(string)0", + EncodedVideoCodec::AV1 => "video/x-av1,stream-format=obu-stream,alignment=tu", + } +} + +/// Returns the appsink caps for a codec. +pub fn encoded_caps(codec: EncodedVideoCodec) -> Result { + encoded_caps_string(codec) + .parse::() + .map_err(|err| GStreamerPipelineError::Pipeline(format!("invalid encoded caps: {err}"))) +} + +/// Returns the appsink sample format used to ingest a codec. +fn sample_format_for_codec(codec: EncodedVideoCodec) -> GStreamerSampleFormat { + match codec { + EncodedVideoCodec::H264 => GStreamerSampleFormat::H264AnnexB, + EncodedVideoCodec::H265 => GStreamerSampleFormat::H265AnnexB, + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 | EncodedVideoCodec::AV1 => { + GStreamerSampleFormat::AccessUnit { codec } + } + } +} + +/// Returns the GStreamer parser element name for a codec, when one is +/// needed. +pub fn parser_name(codec: EncodedVideoCodec) -> Option<&'static str> { + match codec { + EncodedVideoCodec::H264 => Some("h264parse"), + EncodedVideoCodec::H265 => Some("h265parse"), + EncodedVideoCodec::VP8 | EncodedVideoCodec::VP9 => None, + EncodedVideoCodec::AV1 => Some("av1parse"), + } +} + +/// Finds or builds the encoded appsink in a pipeline. +/// +/// When the pipeline already contains `appsink name=lk_appsink`, it is used +/// as-is (its sink caps decide the sample format). Otherwise the pipeline +/// must leave one encoded video source pad unlinked; the codec parser, a +/// capsfilter, and an appsink are created and linked to it. +fn ensure_encoded_appsink( + pipeline: &gst::Pipeline, + requested_codec: Option, +) -> Result<(gst_app::AppSink, GStreamerSampleFormat), GStreamerPipelineError> { + if let Some(appsink) = pipeline.by_name(ENCODED_APPSINK_NAME) { + let sample_format = match sample_format_from_element_sink_caps(&appsink)? { + Some(sample_format) => { + if let Some(requested_codec) = requested_codec { + if requested_codec != sample_format.codec() { + return Err(GStreamerPipelineError::CodecMismatch { + requested: requested_codec, + advertised: sample_format.codec(), + location: format!("appsink '{ENCODED_APPSINK_NAME}'"), + }); + } + } + sample_format + } + None => sample_format_for_codec(requested_codec.unwrap_or(EncodedVideoCodec::H264)), + }; + let appsink = appsink + .downcast::() + .map_err(|_| GStreamerPipelineError::NotAnAppSink)?; + return Ok((appsink, sample_format)); + } + + let src_pad = pipeline + .find_unlinked_pad(gst::PadDirection::Src) + .ok_or(GStreamerPipelineError::MissingAppSink)?; + let inferred_codec = codec_from_pad_caps(&src_pad) + .ok_or_else(|| GStreamerPipelineError::UnsupportedPadCaps(src_pad.name().to_string()))?; + let codec = match requested_codec { + Some(requested_codec) if requested_codec != inferred_codec => { + return Err(GStreamerPipelineError::CodecMismatch { + requested: requested_codec, + advertised: inferred_codec, + location: format!("unlinked pad '{}'", src_pad.name()), + }); + } + Some(requested_codec) => requested_codec, + None => inferred_codec, + }; + let sample_format = sample_format_for_codec(codec); + let src_element = src_pad.parent_element().ok_or_else(|| { + GStreamerPipelineError::Pipeline( + "unlinked GStreamer encoded pad has no parent element".to_owned(), + ) + })?; + + let parser = parser_element_for_codec(codec)?; + let codec_caps = encoded_caps(codec)?; + let capsfilter = gst::ElementFactory::make("capsfilter") + .property("caps", codec_caps) + .build() + .map_err(|err| { + GStreamerPipelineError::Pipeline(format!("failed to create {codec:?} capsfilter: {err}")) + })?; + let appsink = gst::ElementFactory::make("appsink") + .name(ENCODED_APPSINK_NAME) + .property("sync", false) + .property("max-buffers", 8u32) + .property("drop", true) + .build() + .map_err(|err| { + GStreamerPipelineError::Pipeline(format!("failed to create appsink: {err}")) + })?; + + if let Some(parser) = &parser { + pipeline.add(parser).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to add {} to GStreamer pipeline: {err}", + parser.name() + )) + })?; + } + pipeline.add(&capsfilter).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to add capsfilter to GStreamer pipeline: {err}" + )) + })?; + pipeline.add(&appsink).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to add appsink to GStreamer pipeline: {err}" + )) + })?; + if let Some(parser) = &parser { + gst::Element::link_many([parser, &capsfilter, &appsink]).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to link {} to appsink: {err}", + parser.name() + )) + })?; + } else { + gst::Element::link_many([&capsfilter, &appsink]).map_err(|err| { + GStreamerPipelineError::Pipeline(format!("failed to link capsfilter to appsink: {err}")) + })?; + } + let link_target = parser.as_ref().unwrap_or(&capsfilter); + let sink_pad = link_target.static_pad("sink").ok_or_else(|| { + GStreamerPipelineError::Pipeline(format!( + "{} did not expose a sink pad", + link_target.name() + )) + })?; + src_pad.link(&sink_pad).map_err(|err| { + GStreamerPipelineError::Pipeline(format!( + "failed to link '{}' to {}: {err}", + src_element.name(), + link_target.name() + )) + })?; + + let appsink = + appsink.downcast::().map_err(|_| GStreamerPipelineError::NotAnAppSink)?; + Ok((appsink, sample_format)) +} + +fn parser_element_for_codec( + codec: EncodedVideoCodec, +) -> Result, GStreamerPipelineError> { + let Some(name) = parser_name(codec) else { + return Ok(None); + }; + let mut builder = gst::ElementFactory::make(name); + if matches!(codec, EncodedVideoCodec::H264 | EncodedVideoCodec::H265) { + builder = builder.property("config-interval", -1i32); + } + builder + .build() + .map(Some) + .map_err(|err| GStreamerPipelineError::Pipeline(format!("failed to create {name}: {err}"))) +} + +fn sample_format_from_element_sink_caps( + element: &gst::Element, +) -> Result, GStreamerPipelineError> { + let Some(sink_pad) = element.static_pad("sink") else { + return Ok(None); + }; + sample_format_from_pad_caps(&sink_pad) +} + +fn sample_format_from_pad_caps( + pad: &gst::Pad, +) -> Result, GStreamerPipelineError> { + let caps = pad.current_caps().unwrap_or_else(|| pad.query_caps(None)); + for structure in caps.iter() { + if let Some(sample_format) = sample_format_from_caps_structure(structure)? { + return Ok(Some(sample_format)); + } + } + Ok(None) +} + +/// Infers the appsink sample format from a caps structure. +fn sample_format_from_caps_structure( + structure: &gst::StructureRef, +) -> Result, GStreamerPipelineError> { + let Some(codec) = codec_from_caps_name(structure.name()) else { + return Ok(None); + }; + + match codec { + EncodedVideoCodec::H264 => { + let stream_format = structure.get::("stream-format").ok(); + match stream_format.as_deref() { + Some("avc") | Some("avc3") => Ok(Some(GStreamerSampleFormat::H264Avc { + nal_length_size: h264_avc_nal_length_size_from_caps(structure), + })), + Some("byte-stream") | None => Ok(Some(GStreamerSampleFormat::H264AnnexB)), + Some(stream_format) => Err(GStreamerPipelineError::UnsupportedCaps(format!( + "H.264 stream-format '{stream_format}'; expected byte-stream or avc" + ))), + } + } + EncodedVideoCodec::H265 => Ok(Some(GStreamerSampleFormat::H265AnnexB)), + EncodedVideoCodec::VP8 => Ok(Some(GStreamerSampleFormat::AccessUnit { codec })), + EncodedVideoCodec::VP9 => { + let profile = structure.get::("profile").ok(); + match profile.as_deref() { + Some("0") | None => Ok(Some(GStreamerSampleFormat::AccessUnit { codec })), + Some(profile) => Err(GStreamerPipelineError::UnsupportedCaps(format!( + "VP9 profile '{profile}'; expected profile 0" + ))), + } + } + EncodedVideoCodec::AV1 => { + let stream_format = structure.get::("stream-format").ok(); + match stream_format.as_deref() { + Some("obu-stream") | None => Ok(Some(GStreamerSampleFormat::AccessUnit { codec })), + Some(stream_format) => Err(GStreamerPipelineError::UnsupportedCaps(format!( + "AV1 stream-format '{stream_format}'; expected obu-stream" + ))), + } + } + } +} + +fn h264_avc_nal_length_size_from_caps(structure: &gst::StructureRef) -> u8 { + let Ok(codec_data) = structure.get::("codec_data") else { + return 4; + }; + let Ok(codec_data) = codec_data.map_readable() else { + return 4; + }; + h264_avc_nal_length_size_from_codec_data(codec_data.as_ref()).unwrap_or(4) +} + +/// Reads the AVC NAL length-prefix size from `avcC` codec data. +fn h264_avc_nal_length_size_from_codec_data(codec_data: &[u8]) -> Option { + let length_size = (codec_data.get(4)? & 0x03) + 1; + (1..=4).contains(&length_size).then_some(length_size) +} + +/// Infers the encoded codec advertised by a pad's caps. +fn codec_from_pad_caps(pad: &gst::Pad) -> Option { + let caps = pad.current_caps().unwrap_or_else(|| pad.query_caps(None)); + caps.iter().find_map(|structure| codec_from_caps_name(structure.name())) +} + +/// Maps a caps media-type name to an encoded codec. +fn codec_from_caps_name(name: &str) -> Option { + match name { + "video/x-h264" => Some(EncodedVideoCodec::H264), + "video/x-h265" => Some(EncodedVideoCodec::H265), + "video/x-vp8" => Some(EncodedVideoCodec::VP8), + "video/x-vp9" => Some(EncodedVideoCodec::VP9), + "video/x-av1" => Some(EncodedVideoCodec::AV1), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sample_payload_h264_annex_b_detects_keyframe() { + let access_unit = access_unit_from_sample_payload( + GStreamerSampleFormat::H264AnnexB, + &[0, 0, 1, 0x65, 1, 2], + 1_000, + EncodedFrameType::Delta, + VideoResolution::new(640, 480), + ) + .unwrap(); + + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.timestamp_us, 1_000); + } + + #[test] + fn sample_payload_h264_avc_converts_to_annex_b_and_detects_keyframe() { + let access_unit = access_unit_from_sample_payload( + GStreamerSampleFormat::H264Avc { nal_length_size: 4 }, + &[0, 0, 0, 3, 0x65, 1, 2], + 1_000, + EncodedFrameType::Delta, + VideoResolution::new(640, 480), + ) + .unwrap(); + + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + } + + #[test] + fn sample_payload_access_unit_uses_buffer_delta_flag() { + let access_unit = access_unit_from_sample_payload( + GStreamerSampleFormat::AccessUnit { codec: EncodedVideoCodec::VP8 }, + &[1, 2, 3], + 2_000, + EncodedFrameType::Delta, + VideoResolution::new(640, 480), + ) + .unwrap(); + + assert_eq!(access_unit.codec, EncodedVideoCodec::VP8); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + } + + #[test] + fn clock_time_is_offset_from_start_timestamp() { + let timestamp = clock_time_to_timestamp_us(10_000, gst::ClockTime::from_useconds(1_234)); + assert_eq!(timestamp, 11_234); + } +} diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs new file mode 100644 index 000000000..eca6dc31a --- /dev/null +++ b/livekit-capture/src/sources/mod.rs @@ -0,0 +1,28 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Ready-made capture sources. Each source is gated behind its own +//! `source-*` feature. + +#[cfg(feature = "source-clock")] +pub mod clock; + +#[cfg(feature = "source-device")] +pub mod device; + +#[cfg(feature = "source-gstreamer")] +pub mod gstreamer; + +#[cfg(feature = "source-pattern")] +pub mod pattern; diff --git a/livekit-capture/src/sources/pattern.rs b/livekit-capture/src/sources/pattern.rs new file mode 100644 index 000000000..1604f48d9 --- /dev/null +++ b/livekit-capture/src/sources/pattern.rs @@ -0,0 +1,320 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test pattern video source. +//! +//! [`PatternVideoSource`] renders a built-in test pattern ([`Pattern`]) +//! on the GPU and yields the result as pixel video. Rendering is +//! offscreen through [wgpu], so the source needs no window or display. +//! +//! The source reads each frame back from the GPU and converts it to I420 +//! on the CPU. +//! +//! [wgpu]: https://wgpu.rs + +use crate::{ + error::SourceError, + pixel::PixelVideoSource, + primitive::VideoResolution, + pump::PumpStop, + renderer::{FramePacer, RendererError, ShaderRenderer}, +}; +use livekit::webrtc::video_frame::{BoxVideoFrame, VideoFrame, VideoRotation}; +use std::{fmt, time::Duration}; +use thiserror::Error; + +/// Size of the uniform block: `vec2` + `f32` + `u32`. +const UNIFORM_BUFFER_SIZE: u64 = 16; + +/// Period after which the shader time uniform wraps: 2^13 seconds, +/// about 2.3 hours. +/// +/// f32 seconds lose precision as they grow. The wrap keeps the time +/// resolution finer than one millisecond on long runs, at the cost of +/// one pattern discontinuity per period. Frame timestamps do not wrap. +const TIME_WRAP_PERIOD_US: u64 = 8_192_000_000; + +/// Prelude prepended to every fragment snippet. It draws one triangle +/// that covers the full target and calls `shade` per pixel. +const FRAGMENT_PRELUDE: &str = include_str!("../../shaders/prelude.wgsl"); + +/// Fragment snippet for [`Pattern::Gradient`]. +const GRADIENT_SHADER: &str = include_str!("../../shaders/gradient.wgsl"); + +/// Fragment snippet for [`Pattern::Logo`]. +const LOGO_SHADER: &str = include_str!("../../shaders/logo.wgsl"); + +/// Test pattern rendered by a [`PatternVideoSource`]. +/// +/// Every pattern is a pure function of position, resolution, and time: +/// the same configuration produces the same frames on every machine. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(rename_all = "snake_case") +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[non_exhaustive] +pub enum Pattern { + /// Animated color gradient. + Gradient, + /// Bouncing LiveKit logo. + Logo, +} + +impl Pattern { + /// Returns the complete WGSL module to compile. + fn module_code(&self) -> String { + match self { + Self::Gradient => assemble_module(GRADIENT_SHADER), + Self::Logo => assemble_module(LOGO_SHADER), + } + } +} + +/// Prepends the prelude to a pattern's fragment snippet. +fn assemble_module(snippet: &str) -> String { + format!("{FRAGMENT_PRELUDE}\n{snippet}") +} + +/// Configuration for a [`PatternVideoSource`]. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct PatternVideoSourceConfig { + /// Output resolution. + pub resolution: VideoResolution, + /// Output frame rate in frames per second. + pub framerate_fps: u32, + /// Pattern to render. + pub pattern: Pattern, +} + +/// Error returned by a pattern video source. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum PatternVideoSourceError { + /// The configured resolution has a zero component. + #[error("pattern source resolution must be non-zero")] + ZeroResolution, + /// The configured frame rate is zero. + #[error("pattern source frame rate must be non-zero")] + ZeroFramerate, + /// The GPU renderer failed. + #[error(transparent)] + Render(#[from] RendererError), +} + +/// Pixel video source that renders a test pattern on the GPU. +/// +/// The source sleeps to pace itself to the configured frame rate. It +/// never reaches the end of its stream — stop the pump that drives it +/// instead. +pub struct PatternVideoSource { + config: PatternVideoSourceConfig, + renderer: ShaderRenderer, + pacer: FramePacer, +} + +impl PatternVideoSource { + /// Creates the source. GPU setup runs on the tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`PatternVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: PatternVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await + } + + /// Selects a GPU adapter, compiles the pattern's shader, and builds + /// the render pipeline. + /// + /// Construction fails when no GPU is available, or for a zero + /// resolution or frame rate. + pub fn new_blocking(config: PatternVideoSourceConfig) -> Result { + validate_config(&config).map_err(SourceError::new)?; + let renderer = ShaderRenderer::new( + config.resolution, + &config.pattern.module_code(), + UNIFORM_BUFFER_SIZE, + ) + .map_err(|error| SourceError::new(PatternVideoSourceError::Render(error)))?; + let pacer = FramePacer::new(config.framerate_fps); + Ok(Self { config, renderer, pacer }) + } + + /// Returns the configuration the source was created with. + pub fn config(&self) -> &PatternVideoSourceConfig { + &self.config + } +} + +impl PixelVideoSource for PatternVideoSource { + fn resolution(&self) -> VideoResolution { + self.config.resolution + } + + // The pacing sleep is at most one frame interval, and the renderer + // bounds every readback wait, so the stop token is observed promptly. + fn next_frame(&mut self, stop: &PumpStop) -> Result, SourceError> { + let (elapsed, frame_index) = self.pacer.wait_for_next_frame(); + + // The shader time wraps to keep its f32 precision on long runs, + // and the uniform frame index wraps after u32::MAX frames. + let elapsed_us = elapsed.as_micros() as u64; + let time_s = Duration::from_micros(elapsed_us % TIME_WRAP_PERIOD_US).as_secs_f32(); + let uniform = uniform_bytes(self.config.resolution, time_s, frame_index as u32); + + let buffer = self + .renderer + .render_frame(&uniform, stop) + .map_err(|error| SourceError::new(PatternVideoSourceError::Render(error)))?; + let Some(buffer) = buffer else { + // The stop token fired during the readback wait. + return Ok(None); + }; + + Ok(Some(VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: elapsed.as_micros() as i64, + frame_metadata: None, + buffer: Box::new(buffer), + })) + } +} + +impl fmt::Debug for PatternVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PatternVideoSource").field("config", &self.config).finish_non_exhaustive() + } +} + +/// Validates the CPU-checkable parts of a configuration. +fn validate_config(config: &PatternVideoSourceConfig) -> Result<(), PatternVideoSourceError> { + let VideoResolution { width, height } = config.resolution; + if width == 0 || height == 0 { + return Err(PatternVideoSourceError::ZeroResolution); + } + if config.framerate_fps == 0 { + return Err(PatternVideoSourceError::ZeroFramerate); + } + Ok(()) +} + +/// Serializes the uniform block: resolution, time, and frame index. +fn uniform_bytes( + resolution: VideoResolution, + time_s: f32, + frame_index: u32, +) -> [u8; UNIFORM_BUFFER_SIZE as usize] { + let mut bytes = [0u8; UNIFORM_BUFFER_SIZE as usize]; + bytes[0..4].copy_from_slice(&(resolution.width as f32).to_ne_bytes()); + bytes[4..8].copy_from_slice(&(resolution.height as f32).to_ne_bytes()); + bytes[8..12].copy_from_slice(&time_s.to_ne_bytes()); + bytes[12..16].copy_from_slice(&frame_index.to_ne_bytes()); + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::renderer::gpu_available; + + const RESOLUTION: VideoResolution = VideoResolution { width: 64, height: 36 }; + + fn gradient_config() -> PatternVideoSourceConfig { + PatternVideoSourceConfig { + resolution: RESOLUTION, + framerate_fps: 1000, + pattern: Pattern::Gradient, + } + } + + #[test] + fn validation_rejects_zero_resolution_and_framerate() { + let mut config = gradient_config(); + config.resolution = VideoResolution::new(0, 36); + assert!(matches!(validate_config(&config), Err(PatternVideoSourceError::ZeroResolution))); + + let mut config = gradient_config(); + config.framerate_fps = 0; + assert!(matches!(validate_config(&config), Err(PatternVideoSourceError::ZeroFramerate))); + } + + #[test] + fn pattern_modules_include_the_prelude() { + for pattern in [Pattern::Gradient, Pattern::Logo] { + let code = pattern.module_code(); + assert!(code.contains("fn vs_main"), "{pattern:?} is missing the prelude"); + assert!(code.contains("fn fs_main"), "{pattern:?} is missing the prelude"); + assert!(code.contains("fn shade"), "{pattern:?} is missing a shade function"); + } + } + + #[test] + fn gradient_renders_frames_at_the_frame_rate() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut source = PatternVideoSource::new_blocking(gradient_config()).unwrap(); + + let stop = PumpStop::new(); + let first = source.next_frame(&stop).unwrap().unwrap(); + let second = source.next_frame(&stop).unwrap().unwrap(); + assert_eq!((first.buffer.width(), first.buffer.height()), (64, 36)); + assert_eq!(first.timestamp_us, 0); + assert_eq!(second.timestamp_us, 1_000); + + // The gradient's top-left pixel at time zero is red-dominant: + // RGB (255, 68, 47), which is about (121, 91, 211) in + // limited-range BT.601. A red/blue channel swap in the readback + // path flips the two chroma values, so this check catches it. + let i420 = first.buffer.as_i420().expect("pattern source yields I420 buffers"); + let (y, u, v) = i420.data(); + assert!(y[0].abs_diff(121) <= 5, "unexpected luma {}", y[0]); + assert!(u[0].abs_diff(91) <= 6, "unexpected chroma-u {}", u[0]); + assert!(v[0].abs_diff(211) <= 6, "unexpected chroma-v {}", v[0]); + } + + #[test] + fn logo_renders_on_a_black_background() { + if !gpu_available() { + eprintln!("skipping: no GPU adapter available"); + return; + } + let mut source = PatternVideoSource::new_blocking(PatternVideoSourceConfig { + resolution: RESOLUTION, + framerate_fps: 1000, + pattern: Pattern::Logo, + }) + .unwrap(); + + let frame = source.next_frame(&PumpStop::new()).unwrap().unwrap(); + let i420 = frame.buffer.as_i420().expect("pattern source yields I420 buffers"); + let (y, _, _) = i420.data(); + + // The logo starts away from the corners, so the top-left pixel is + // background black (luma 16 in limited range), and the lit tile + // pixels stand out well above it. + assert!(y[0] <= 20, "top-left pixel is not background: {}", y[0]); + let lit = y.iter().filter(|&&luma| luma > 60).count(); + assert!(lit > 5, "no logo pixels found (lit count {lit})"); + } +} diff --git a/livekit-capture/src/utils.rs b/livekit-capture/src/utils.rs new file mode 100644 index 000000000..4a6e070ac --- /dev/null +++ b/livekit-capture/src/utils.rs @@ -0,0 +1,29 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Crate-internal utilities. + +/// Runs a blocking task on the tokio blocking pool. Panics resume on the +/// caller, and join failures become source errors. +#[cfg(feature = "tokio")] +#[allow(dead_code)] +pub(crate) async fn run_blocking( + task: impl FnOnce() -> Result + Send + 'static, +) -> Result { + match tokio::task::spawn_blocking(task).await { + Ok(result) => result, + Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), + Err(err) => Err(crate::error::SourceError::new(err)), + } +} diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts index ac81cb93e..0ffa8cb94 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts @@ -28,6 +28,7 @@ import type { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, Registe import type { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } from "./track_publication_pb.js"; import type { ByteStreamOpenCallback, ByteStreamOpenRequest, ByteStreamOpenResponse, ByteStreamReaderEvent, ByteStreamReaderReadAllCallback, ByteStreamReaderReadAllRequest, ByteStreamReaderReadAllResponse, ByteStreamReaderReadIncrementalRequest, ByteStreamReaderReadIncrementalResponse, ByteStreamReaderWriteToFileCallback, ByteStreamReaderWriteToFileRequest, ByteStreamReaderWriteToFileResponse, ByteStreamWriterCloseCallback, ByteStreamWriterCloseRequest, ByteStreamWriterCloseResponse, ByteStreamWriterWriteCallback, ByteStreamWriterWriteRequest, ByteStreamWriterWriteResponse, StreamSendBytesCallback, StreamSendBytesRequest, StreamSendBytesResponse, StreamSendFileCallback, StreamSendFileRequest, StreamSendFileResponse, StreamSendTextCallback, StreamSendTextRequest, StreamSendTextResponse, TextStreamOpenCallback, TextStreamOpenRequest, TextStreamOpenResponse, TextStreamReaderEvent, TextStreamReaderReadAllCallback, TextStreamReaderReadAllRequest, TextStreamReaderReadAllResponse, TextStreamReaderReadIncrementalRequest, TextStreamReaderReadIncrementalResponse, TextStreamWriterCloseCallback, TextStreamWriterCloseRequest, TextStreamWriterCloseResponse, TextStreamWriterWriteCallback, TextStreamWriterWriteRequest, TextStreamWriterWriteResponse } from "./data_stream_pb.js"; import type { DataTrackStreamEvent, DataTrackStreamReadRequest, DataTrackStreamReadResponse, DefineSchemaCallback, DefineSchemaRequest, DefineSchemaResponse, GetSchemaCallback, GetSchemaRequest, GetSchemaResponse, LocalDataTrackIsPublishedRequest, LocalDataTrackIsPublishedResponse, LocalDataTrackTryPushRequest, LocalDataTrackTryPushResponse, LocalDataTrackUnpublishRequest, LocalDataTrackUnpublishResponse, PublishDataTrackCallback, PublishDataTrackRequest, PublishDataTrackResponse, RemoteDataTrackIsPublishedRequest, RemoteDataTrackIsPublishedResponse, RemoteDataTrackSetPipelineOptionsRequest, RemoteDataTrackSetPipelineOptionsResponse, SubscribeDataTrackRequest, SubscribeDataTrackResponse } from "./data_track_pb.js"; +import type { CaptureSourceEvent, ListCaptureDevicesCallback, ListCaptureDevicesRequest, ListCaptureDevicesResponse, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } from "./capture_pb.js"; /** * @generated from enum livekit.proto.LogLevel @@ -611,6 +612,32 @@ export declare class FfiRequest extends Message { */ value: ReadyForRoomEventRequest; case: "readyForRoomEvent"; + } | { + /** + * Capture sources (livekit-capture; requires the `capture` feature) + * + * @generated from field: livekit.proto.NewCaptureSourceRequest new_capture_source = 87; + */ + value: NewCaptureSourceRequest; + case: "newCaptureSource"; + } | { + /** + * @generated from field: livekit.proto.StartCaptureRequest start_capture = 88; + */ + value: StartCaptureRequest; + case: "startCapture"; + } | { + /** + * @generated from field: livekit.proto.StopCaptureRequest stop_capture = 89; + */ + value: StopCaptureRequest; + case: "stopCapture"; + } | { + /** + * @generated from field: livekit.proto.ListCaptureDevicesRequest list_capture_devices = 90; + */ + value: ListCaptureDevicesRequest; + case: "listCaptureDevices"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); @@ -1173,6 +1200,32 @@ export declare class FfiResponse extends Message { */ value: ReadyForRoomEventResponse; case: "readyForRoomEvent"; + } | { + /** + * Capture sources (livekit-capture; requires the `capture` feature) + * + * @generated from field: livekit.proto.NewCaptureSourceResponse new_capture_source = 87; + */ + value: NewCaptureSourceResponse; + case: "newCaptureSource"; + } | { + /** + * @generated from field: livekit.proto.StartCaptureResponse start_capture = 88; + */ + value: StartCaptureResponse; + case: "startCapture"; + } | { + /** + * @generated from field: livekit.proto.StopCaptureResponse stop_capture = 89; + */ + value: StopCaptureResponse; + case: "stopCapture"; + } | { + /** + * @generated from field: livekit.proto.ListCaptureDevicesResponse list_capture_devices = 90; + */ + value: ListCaptureDevicesResponse; + case: "listCaptureDevices"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); @@ -1481,6 +1534,26 @@ export declare class FfiEvent extends Message { */ value: GetSchemaCallback; case: "getSchema"; + } | { + /** + * Capture sources (livekit-capture; requires the `capture` feature) + * + * @generated from field: livekit.proto.NewCaptureSourceCallback new_capture_source = 47; + */ + value: NewCaptureSourceCallback; + case: "newCaptureSource"; + } | { + /** + * @generated from field: livekit.proto.CaptureSourceEvent capture_source_event = 48; + */ + value: CaptureSourceEvent; + case: "captureSourceEvent"; + } | { + /** + * @generated from field: livekit.proto.ListCaptureDevicesCallback list_capture_devices = 49; + */ + value: ListCaptureDevicesCallback; + case: "listCaptureDevices"; } | { case: undefined; value?: undefined }; constructor(data?: PartialMessage); diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.js b/livekit-ffi-node-bindings/proto/ffi_pb.js index 4f0bdd422..0755806d5 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.js +++ b/livekit-ffi-node-bindings/proto/ffi_pb.js @@ -30,6 +30,7 @@ const { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMe const { EnableRemoteTrackPublicationRequest, EnableRemoteTrackPublicationResponse, SetRemoteTrackPublicationQualityRequest, SetRemoteTrackPublicationQualityResponse, UpdateRemoteTrackPublicationDimensionRequest, UpdateRemoteTrackPublicationDimensionResponse } = require("./track_publication_pb.js"); const { ByteStreamOpenCallback, ByteStreamOpenRequest, ByteStreamOpenResponse, ByteStreamReaderEvent, ByteStreamReaderReadAllCallback, ByteStreamReaderReadAllRequest, ByteStreamReaderReadAllResponse, ByteStreamReaderReadIncrementalRequest, ByteStreamReaderReadIncrementalResponse, ByteStreamReaderWriteToFileCallback, ByteStreamReaderWriteToFileRequest, ByteStreamReaderWriteToFileResponse, ByteStreamWriterCloseCallback, ByteStreamWriterCloseRequest, ByteStreamWriterCloseResponse, ByteStreamWriterWriteCallback, ByteStreamWriterWriteRequest, ByteStreamWriterWriteResponse, StreamSendBytesCallback, StreamSendBytesRequest, StreamSendBytesResponse, StreamSendFileCallback, StreamSendFileRequest, StreamSendFileResponse, StreamSendTextCallback, StreamSendTextRequest, StreamSendTextResponse, TextStreamOpenCallback, TextStreamOpenRequest, TextStreamOpenResponse, TextStreamReaderEvent, TextStreamReaderReadAllCallback, TextStreamReaderReadAllRequest, TextStreamReaderReadAllResponse, TextStreamReaderReadIncrementalRequest, TextStreamReaderReadIncrementalResponse, TextStreamWriterCloseCallback, TextStreamWriterCloseRequest, TextStreamWriterCloseResponse, TextStreamWriterWriteCallback, TextStreamWriterWriteRequest, TextStreamWriterWriteResponse } = require("./data_stream_pb.js"); const { DataTrackStreamEvent, DataTrackStreamReadRequest, DataTrackStreamReadResponse, DefineSchemaCallback, DefineSchemaRequest, DefineSchemaResponse, GetSchemaCallback, GetSchemaRequest, GetSchemaResponse, LocalDataTrackIsPublishedRequest, LocalDataTrackIsPublishedResponse, LocalDataTrackTryPushRequest, LocalDataTrackTryPushResponse, LocalDataTrackUnpublishRequest, LocalDataTrackUnpublishResponse, PublishDataTrackCallback, PublishDataTrackRequest, PublishDataTrackResponse, RemoteDataTrackIsPublishedRequest, RemoteDataTrackIsPublishedResponse, RemoteDataTrackSetPipelineOptionsRequest, RemoteDataTrackSetPipelineOptionsResponse, SubscribeDataTrackRequest, SubscribeDataTrackResponse } = require("./data_track_pb.js"); +const { CaptureSourceEvent, ListCaptureDevicesCallback, ListCaptureDevicesRequest, ListCaptureDevicesResponse, NewCaptureSourceCallback, NewCaptureSourceRequest, NewCaptureSourceResponse, StartCaptureRequest, StartCaptureResponse, StopCaptureRequest, StopCaptureResponse } = require("./capture_pb.js"); /** * @generated from enum livekit.proto.LogLevel @@ -139,6 +140,10 @@ const FfiRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 81, name: "start_recording", kind: "message", T: StartRecordingRequest, oneof: "message" }, { no: 82, name: "stop_recording", kind: "message", T: StopRecordingRequest, oneof: "message" }, { no: 83, name: "ready_for_room_event", kind: "message", T: ReadyForRoomEventRequest, oneof: "message" }, + { no: 87, name: "new_capture_source", kind: "message", T: NewCaptureSourceRequest, oneof: "message" }, + { no: 88, name: "start_capture", kind: "message", T: StartCaptureRequest, oneof: "message" }, + { no: 89, name: "stop_capture", kind: "message", T: StopCaptureRequest, oneof: "message" }, + { no: 90, name: "list_capture_devices", kind: "message", T: ListCaptureDevicesRequest, oneof: "message" }, ], ); @@ -234,6 +239,10 @@ const FfiResponse = /*@__PURE__*/ proto2.makeMessageType( { no: 80, name: "start_recording", kind: "message", T: StartRecordingResponse, oneof: "message" }, { no: 81, name: "stop_recording", kind: "message", T: StopRecordingResponse, oneof: "message" }, { no: 82, name: "ready_for_room_event", kind: "message", T: ReadyForRoomEventResponse, oneof: "message" }, + { no: 87, name: "new_capture_source", kind: "message", T: NewCaptureSourceResponse, oneof: "message" }, + { no: 88, name: "start_capture", kind: "message", T: StartCaptureResponse, oneof: "message" }, + { no: 89, name: "stop_capture", kind: "message", T: StopCaptureResponse, oneof: "message" }, + { no: 90, name: "list_capture_devices", kind: "message", T: ListCaptureDevicesResponse, oneof: "message" }, ], ); @@ -292,6 +301,9 @@ const FfiEvent = /*@__PURE__*/ proto2.makeMessageType( { no: 44, name: "simulate_scenario", kind: "message", T: SimulateScenarioCallback, oneof: "message" }, { no: 45, name: "define_schema", kind: "message", T: DefineSchemaCallback, oneof: "message" }, { no: 46, name: "get_schema", kind: "message", T: GetSchemaCallback, oneof: "message" }, + { no: 47, name: "new_capture_source", kind: "message", T: NewCaptureSourceCallback, oneof: "message" }, + { no: 48, name: "capture_source_event", kind: "message", T: CaptureSourceEvent, oneof: "message" }, + { no: 49, name: "list_capture_devices", kind: "message", T: ListCaptureDevicesCallback, oneof: "message" }, ], ); diff --git a/livekit-ffi-node-bindings/proto/room_pb.d.ts b/livekit-ffi-node-bindings/proto/room_pb.d.ts index 069ca8f5c..8f37187a6 100644 --- a/livekit-ffi-node-bindings/proto/room_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/room_pb.d.ts @@ -125,6 +125,13 @@ export declare enum VideoEncoderBackend { * @generated from enum value: ENCODER_BACKEND_VIDEOTOOLBOX = 5; */ ENCODER_BACKEND_VIDEOTOOLBOX = 5, + + /** + * Pre-encoded passthrough: the application supplies encoded frames. + * + * @generated from enum value: ENCODER_BACKEND_PRE_ENCODED = 6; + */ + ENCODER_BACKEND_PRE_ENCODED = 6, } /** diff --git a/livekit-ffi-node-bindings/proto/room_pb.js b/livekit-ffi-node-bindings/proto/room_pb.js index 6a6a5ed44..539db79d9 100644 --- a/livekit-ffi-node-bindings/proto/room_pb.js +++ b/livekit-ffi-node-bindings/proto/room_pb.js @@ -65,6 +65,7 @@ const VideoEncoderBackend = /*@__PURE__*/ proto2.makeEnum( {no: 3, name: "ENCODER_BACKEND_NVENC"}, {no: 4, name: "ENCODER_BACKEND_VAAPI"}, {no: 5, name: "ENCODER_BACKEND_VIDEOTOOLBOX"}, + {no: 6, name: "ENCODER_BACKEND_PRE_ENCODED"}, ], ); diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index a48df84d4..525761e23 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -19,8 +19,21 @@ __rustls-tls = ["livekit/__rustls-tls"] # Enable tokio-console to debug tasks tracing = ["tokio/tracing", "console-subscriber"] +# Capture sources (livekit-capture): publish tracks from server-side +# producers such as camera devices or GStreamer pipelines. Links system +# GStreamer. +capture = [ + "dep:livekit-capture", + "livekit-capture/source-clock", + "livekit-capture/source-device", + "livekit-capture/source-gstreamer", + "livekit-capture/source-pattern", + "livekit-capture/tokio", +] + [dependencies] livekit = { workspace = true } +livekit-capture = { workspace = true, optional = true, default-features = false } webrtc-sys = { workspace = true } soxr-sys = { workspace = true } imgproc = { workspace = true } diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto new file mode 100644 index 000000000..4a5099e55 --- /dev/null +++ b/livekit-ffi/protocol/capture.proto @@ -0,0 +1,279 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package livekit.proto; +option csharp_namespace = "LiveKit.Proto"; + +import "handle.proto"; +import "room.proto"; +import "video_frame.proto"; + +// Capture sources (livekit-capture) +// +// A capture source owns a media producer (e.g. a GStreamer pipeline) and the +// pump that forwards its frames into an RTC video source, so no per-frame +// FFI traffic is needed. These requests are only served when the FFI server +// is built with the `capture` feature; otherwise they fail with an error. +// +// Usage: +// 1. NewCaptureSourceRequest (async): builds and starts the producer, +// discovers stream settings, and returns an OwnedCaptureSource. Its info +// embeds an OwnedVideoSource for the existing CreateVideoTrackRequest and +// the recommended TrackPublishOptions for PublishTrackRequest. +// 2. StartCaptureRequest: starts pumping frames into the RTC source. +// 3. The capture runs until StopCaptureRequest, end of stream, or an error; +// a CaptureSourceEvent is delivered exactly once when it ends. + +// Bitrate unit expected by a GStreamer encoder property. +enum GstreamerBitrateUnit { + GSTREAMER_BITRATE_UNIT_BPS = 0; + GSTREAMER_BITRATE_UNIT_KBPS = 1; +} + +// Binding from WebRTC rate-control targets to a GStreamer encoder property. +message GstreamerRateControl { + // Name of the encoder element in the pipeline (e.g. `lk_encoder`). + required string element = 1; + // Bitrate property to set on the element (e.g. `bitrate` for x264enc, + // `target-bitrate` for vp8enc/vp9enc). + required string property = 2; + // Unit the property expects. + required GstreamerBitrateUnit unit = 3; +} + +// Encoded ingest from a GStreamer pipeline. +message GstreamerVideoSourceConfig { + // GStreamer launch description for the encoded producer pipeline. + // Must contain `appsink name=lk_appsink`, or leave exactly one encoded + // video source pad unlinked for the source to attach one to. + required string pipeline = 1; + // Codec expected from the pipeline; inferred from pipeline caps when + // omitted. + optional VideoCodec codec = 2; + // Encoded frame resolution. When omitted, it is discovered from the + // pipeline's negotiated caps; when set, the pipeline output is verified + // against it. + optional VideoSourceResolution resolution = 3; + // Forwards WebRTC rate-control targets to an encoder element's bitrate + // property. Without this, the pipeline encodes at a fixed bitrate. + optional GstreamerRateControl rate_control = 4; +} + +// Test patterns built into livekit-capture. +enum Pattern { + // Animated color gradient. + PATTERN_GRADIENT = 0; + // Bouncing LiveKit logo. + PATTERN_LOGO = 1; +} + +// Test pattern rendered on the GPU. +message PatternVideoSourceConfig { + // Output resolution. + required VideoSourceResolution resolution = 1; + // Output frame rate in frames per second. + required uint32 framerate_fps = 2; + // Pattern to render. + required Pattern pattern = 3; +} + +// Wall clock with millisecond precision, rendered on the GPU. Shows the +// local time of the machine that runs the FFI server. +message ClockVideoSourceConfig { + // Output resolution. + required VideoSourceResolution resolution = 1; + // Output frame rate in frames per second. + required uint32 framerate_fps = 2; +} + +// Frame format delivered by a capture device. +enum DeviceFrameFormat { + DEVICE_FRAME_FORMAT_I420 = 0; + DEVICE_FRAME_FORMAT_NV12 = 1; + DEVICE_FRAME_FORMAT_BGRA = 2; + DEVICE_FRAME_FORMAT_RGB24 = 3; + DEVICE_FRAME_FORMAT_BGR24 = 4; + DEVICE_FRAME_FORMAT_YUYV = 5; + DEVICE_FRAME_FORMAT_UYVY = 6; + DEVICE_FRAME_FORMAT_GREY = 7; + DEVICE_FRAME_FORMAT_MJPEG = 8; +} + +// Capture format offered by or requested from a device. +message DeviceFormat { + // Frame dimensions. + required VideoSourceResolution resolution = 1; + // Frame rate in frames per second. + required uint32 framerate_fps = 2; + // Frame format. + required DeviceFrameFormat frame_format = 3; +} + +// Format selection requested from a capture device. The device negotiates +// the delivered format; CaptureSourceInfo reports the outcome. +message DeviceFormatRequest { + // Prefer the highest frame rate, optionally constrained. + message HighestFramerate { + optional VideoSourceResolution resolution = 1; + optional DeviceFrameFormat frame_format = 2; + } + // Prefer the highest resolution, optionally constrained. + message HighestResolution { + optional uint32 framerate_fps = 1; + optional DeviceFrameFormat frame_format = 2; + } + // The device's default format when unset. + oneof request { + // Require an exact format match. + DeviceFormat exact = 1; + // Use the device's closest supported format. + DeviceFormat closest = 2; + HighestFramerate highest_framerate = 3; + HighestResolution highest_resolution = 4; + } +} + +// Camera device capture using the platform's native capture stack. +message DeviceVideoSourceConfig { + // Device to capture from; the platform default device when unset. + oneof device { + // Position in the platform enumeration order. + uint32 device_index = 1; + // Platform-stable identifier, as reported by CaptureDeviceInfo.id. + string device_id = 2; + } + // Format requested from the device; the device default when unset. + optional DeviceFormatRequest format = 3; +} + +// Video capture device discovered by ListCaptureDevicesRequest. +message CaptureDeviceInfo { + // Platform-stable device identifier. + required string id = 1; + // Human-readable device name. + required string name = 2; + // Device model identifier, when available. + optional string model_id = 3; + // Device manufacturer, when available. + optional string manufacturer = 4; + // Capture formats reported by the device. + repeated DeviceFormat formats = 5; + // Whether `formats` is a complete list; some platforms do not enumerate + // formats up front. + required bool formats_complete = 6; +} + +message CaptureDeviceList { repeated CaptureDeviceInfo devices = 1; } + +// List the video capture devices available on this machine. +// +// Completes asynchronously with a ListCaptureDevicesCallback: enumeration +// queries the platform capture stack and may block briefly. +message ListCaptureDevicesRequest { optional uint64 request_async_id = 1; } +message ListCaptureDevicesResponse { required uint64 async_id = 1; } +message ListCaptureDevicesCallback { + required uint64 async_id = 1; + oneof message { + string error = 2; + CaptureDeviceList devices = 3; + } +} + +// Kind of media a capture source produces. +enum CaptureSourceKind { + // Pixel frames, published through the WebRTC encoder. + CAPTURE_SOURCE_PIXEL = 0; + // Pre-encoded access units, published as passthrough. + CAPTURE_SOURCE_ENCODED = 1; +} + +message CaptureSourceInfo { + required CaptureSourceKind kind = 1; + // Declared or discovered stream resolution. + required VideoSourceResolution resolution = 2; + // Codec produced by the source; encoded sources only. + optional VideoCodec codec = 3; + // Publish options derived from the source (codec, encoder backend, ...). + // Merge application options (e.g. track source) over these when calling + // PublishTrackRequest. + required TrackPublishOptions recommended_publish_options = 4; + // RTC video source fed by this capture source; pass its handle to + // CreateVideoTrackRequest. Owned by the client like any other handle. + required OwnedVideoSource video_source = 5; +} + +message OwnedCaptureSource { + required FfiOwnedHandle handle = 1; + required CaptureSourceInfo info = 2; +} + +// Create a new capture source from configuration. +// +// Completes asynchronously with a NewCaptureSourceCallback: construction +// starts the producer and may wait for its first output to discover stream +// settings. +message NewCaptureSourceRequest { + oneof config { + GstreamerVideoSourceConfig gstreamer = 1; + PatternVideoSourceConfig pattern = 2; + DeviceVideoSourceConfig device = 4; + ClockVideoSourceConfig clock = 5; + } + optional uint64 request_async_id = 3; +} +message NewCaptureSourceResponse { required uint64 async_id = 1; } +message NewCaptureSourceCallback { + required uint64 async_id = 1; + oneof message { + string error = 2; + OwnedCaptureSource source = 3; + } +} + +// Start pumping frames from a capture source into its RTC video source. +message StartCaptureRequest { required uint64 capture_handle = 1; } +message StartCaptureResponse { optional string error = 1; } + +// Signal a running capture to stop after the frame in flight. The terminal +// CaptureSourceEvent follows shortly. Stopping an already-finished capture +// is a no-op. +message StopCaptureRequest { required uint64 capture_handle = 1; } +message StopCaptureResponse { optional string error = 1; } + +// Why a capture ended without error. +enum CaptureExit { + // Stopped by StopCaptureRequest (or handle disposal). + CAPTURE_EXIT_STOPPED = 0; + // The producer reached the end of its stream. + CAPTURE_EXIT_END_OF_STREAM = 1; +} + +message CaptureFinished { + required uint64 frames_captured = 1; + required CaptureExit exit = 2; +} + +message CaptureError { required string error = 1; } + +// Delivered exactly once when a started capture ends, regardless of why +// (stop request, end of stream, or failure). +message CaptureSourceEvent { + required uint64 capture_handle = 1; + oneof message { + CaptureFinished finished = 2; + CaptureError error = 3; + } +} diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index 53ea782bd..971801216 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -19,6 +19,7 @@ package livekit.proto; import "audio_frame.proto"; import "data_stream.proto"; import "data_track.proto"; +import "capture.proto"; // import "handle.proto"; import "e2ee.proto"; import "room.proto"; @@ -184,7 +185,13 @@ message FfiRequest { // Room event ready signal ReadyForRoomEventRequest ready_for_room_event = 83; - // NEXT_ID: 87 + // Capture sources (livekit-capture; requires the `capture` feature) + NewCaptureSourceRequest new_capture_source = 87; + StartCaptureRequest start_capture = 88; + StopCaptureRequest stop_capture = 89; + ListCaptureDevicesRequest list_capture_devices = 90; + + // NEXT_ID: 91 } } @@ -313,7 +320,13 @@ message FfiResponse { // Room event ready signal ReadyForRoomEventResponse ready_for_room_event = 82; - // NEXT_ID: 87 + // Capture sources (livekit-capture; requires the `capture` feature) + NewCaptureSourceResponse new_capture_source = 87; + StartCaptureResponse start_capture = 88; + StopCaptureResponse stop_capture = 89; + ListCaptureDevicesResponse list_capture_devices = 90; + + // NEXT_ID: 91 } } @@ -382,7 +395,12 @@ message FfiEvent { DefineSchemaCallback define_schema = 45; GetSchemaCallback get_schema = 46; - // NEXT_ID: 47 + // Capture sources (livekit-capture; requires the `capture` feature) + NewCaptureSourceCallback new_capture_source = 47; + CaptureSourceEvent capture_source_event = 48; + ListCaptureDevicesCallback list_capture_devices = 49; + + // NEXT_ID: 50 } } diff --git a/livekit-ffi/protocol/room.proto b/livekit-ffi/protocol/room.proto index df08ed5be..70df35c4d 100644 --- a/livekit-ffi/protocol/room.proto +++ b/livekit-ffi/protocol/room.proto @@ -335,6 +335,8 @@ enum VideoEncoderBackend { ENCODER_BACKEND_NVENC = 3; ENCODER_BACKEND_VAAPI = 4; ENCODER_BACKEND_VIDEOTOOLBOX = 5; + // Pre-encoded passthrough: the application supplies encoded frames. + ENCODER_BACKEND_PRE_ENCODED = 6; } // Controls how the encoder degrades quality when bandwidth is constrained. diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs new file mode 100644 index 000000000..0d94b36ac --- /dev/null +++ b/livekit-ffi/src/conversion/capture.rs @@ -0,0 +1,224 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{proto, FfiError, FfiResult}; +use livekit_capture::{ + encoded::EncodedVideoCodec, + primitive::VideoResolution, + sources::{ + clock::ClockVideoSourceConfig, + device::{ + DeviceFormat, DeviceFormatRequest, DeviceFrameFormat, DeviceInfo, DeviceSelector, + DeviceVideoSourceConfig, + }, + gstreamer::{GStreamerBitrateUnit, GStreamerRateControlConfig, GStreamerVideoSourceConfig}, + pattern::{Pattern, PatternVideoSourceConfig}, + }, +}; + +impl From for VideoResolution { + fn from(resolution: proto::VideoSourceResolution) -> Self { + Self::new(resolution.width, resolution.height) + } +} + +impl From for ClockVideoSourceConfig { + fn from(config: proto::ClockVideoSourceConfig) -> Self { + Self { resolution: config.resolution.into(), framerate_fps: config.framerate_fps } + } +} + +pub fn pattern_config_from_proto( + config: proto::PatternVideoSourceConfig, +) -> FfiResult { + let pattern = proto::Pattern::try_from(config.pattern) + .map_err(|_| FfiError::InvalidRequest("invalid pattern".into()))?; + Ok(PatternVideoSourceConfig { + resolution: config.resolution.into(), + framerate_fps: config.framerate_fps, + pattern: match pattern { + proto::Pattern::Gradient => Pattern::Gradient, + proto::Pattern::Logo => Pattern::Logo, + }, + }) +} + +impl From for GStreamerBitrateUnit { + fn from(unit: proto::GstreamerBitrateUnit) -> Self { + match unit { + proto::GstreamerBitrateUnit::Bps => Self::BitsPerSecond, + proto::GstreamerBitrateUnit::Kbps => Self::KilobitsPerSecond, + } + } +} + +pub fn video_codec_from_proto(codec: proto::VideoCodec) -> EncodedVideoCodec { + match codec { + proto::VideoCodec::H264 => EncodedVideoCodec::H264, + proto::VideoCodec::H265 => EncodedVideoCodec::H265, + proto::VideoCodec::Vp8 => EncodedVideoCodec::VP8, + proto::VideoCodec::Vp9 => EncodedVideoCodec::VP9, + proto::VideoCodec::Av1 => EncodedVideoCodec::AV1, + } +} + +pub fn video_codec_to_proto(codec: EncodedVideoCodec) -> Option { + match codec { + EncodedVideoCodec::H264 => Some(proto::VideoCodec::H264), + EncodedVideoCodec::H265 => Some(proto::VideoCodec::H265), + EncodedVideoCodec::VP8 => Some(proto::VideoCodec::Vp8), + EncodedVideoCodec::VP9 => Some(proto::VideoCodec::Vp9), + EncodedVideoCodec::AV1 => Some(proto::VideoCodec::Av1), + // The codec enum is non-exhaustive; codecs unknown to the protocol + // are simply not reported. + _ => None, + } +} + +fn device_frame_format_from_proto(format: proto::DeviceFrameFormat) -> DeviceFrameFormat { + match format { + proto::DeviceFrameFormat::I420 => DeviceFrameFormat::I420, + proto::DeviceFrameFormat::Nv12 => DeviceFrameFormat::Nv12, + proto::DeviceFrameFormat::Bgra => DeviceFrameFormat::Bgra, + proto::DeviceFrameFormat::Rgb24 => DeviceFrameFormat::Rgb24, + proto::DeviceFrameFormat::Bgr24 => DeviceFrameFormat::Bgr24, + proto::DeviceFrameFormat::Yuyv => DeviceFrameFormat::Yuyv, + proto::DeviceFrameFormat::Uyvy => DeviceFrameFormat::Uyvy, + proto::DeviceFrameFormat::Grey => DeviceFrameFormat::Grey, + proto::DeviceFrameFormat::Mjpeg => DeviceFrameFormat::Mjpeg, + } +} + +fn device_frame_format_to_proto(format: DeviceFrameFormat) -> Option { + match format { + DeviceFrameFormat::I420 => Some(proto::DeviceFrameFormat::I420), + DeviceFrameFormat::Nv12 => Some(proto::DeviceFrameFormat::Nv12), + DeviceFrameFormat::Bgra => Some(proto::DeviceFrameFormat::Bgra), + DeviceFrameFormat::Rgb24 => Some(proto::DeviceFrameFormat::Rgb24), + DeviceFrameFormat::Bgr24 => Some(proto::DeviceFrameFormat::Bgr24), + DeviceFrameFormat::Yuyv => Some(proto::DeviceFrameFormat::Yuyv), + DeviceFrameFormat::Uyvy => Some(proto::DeviceFrameFormat::Uyvy), + DeviceFrameFormat::Grey => Some(proto::DeviceFrameFormat::Grey), + DeviceFrameFormat::Mjpeg => Some(proto::DeviceFrameFormat::Mjpeg), + // The frame format enum is non-exhaustive; formats unknown to the + // protocol are simply not reported. + _ => None, + } +} + +fn decode_device_frame_format(value: i32) -> FfiResult { + proto::DeviceFrameFormat::try_from(value) + .map(device_frame_format_from_proto) + .map_err(|_| FfiError::InvalidRequest("invalid device frame format".into())) +} + +fn device_format_from_proto(format: proto::DeviceFormat) -> FfiResult { + Ok(DeviceFormat { + resolution: format.resolution.into(), + framerate_fps: format.framerate_fps, + frame_format: decode_device_frame_format(format.frame_format)?, + }) +} + +fn device_format_to_proto(format: DeviceFormat) -> Option { + Some(proto::DeviceFormat { + resolution: proto::VideoSourceResolution { + width: format.resolution.width, + height: format.resolution.height, + }, + framerate_fps: format.framerate_fps, + frame_format: device_frame_format_to_proto(format.frame_format)?.into(), + }) +} + +fn device_format_request_from_proto( + request: proto::DeviceFormatRequest, +) -> FfiResult { + use proto::device_format_request::Request; + Ok(match request.request { + None => DeviceFormatRequest::Default, + Some(Request::Exact(format)) => { + DeviceFormatRequest::Exact(device_format_from_proto(format)?) + } + Some(Request::Closest(format)) => { + DeviceFormatRequest::Closest(device_format_from_proto(format)?) + } + Some(Request::HighestFramerate(constraint)) => DeviceFormatRequest::HighestFramerate { + resolution: constraint.resolution.map(VideoResolution::from), + frame_format: constraint.frame_format.map(decode_device_frame_format).transpose()?, + }, + Some(Request::HighestResolution(constraint)) => DeviceFormatRequest::HighestResolution { + framerate_fps: constraint.framerate_fps, + frame_format: constraint.frame_format.map(decode_device_frame_format).transpose()?, + }, + }) +} + +pub fn device_config_from_proto( + config: proto::DeviceVideoSourceConfig, +) -> FfiResult { + use proto::device_video_source_config::Device; + let device = match config.device { + None => DeviceSelector::Default, + Some(Device::DeviceIndex(index)) => DeviceSelector::Index(index as usize), + Some(Device::DeviceId(id)) => DeviceSelector::Id(id), + }; + let format = + config.format.map(device_format_request_from_proto).transpose()?.unwrap_or_default(); + Ok(DeviceVideoSourceConfig { device, format }) +} + +pub fn device_info_to_proto(info: DeviceInfo) -> proto::CaptureDeviceInfo { + proto::CaptureDeviceInfo { + id: info.id, + name: info.name, + model_id: info.model_id, + manufacturer: info.manufacturer, + formats: info.formats.into_iter().filter_map(device_format_to_proto).collect(), + formats_complete: info.formats_complete, + } +} + +pub fn gstreamer_config_from_proto( + config: proto::GstreamerVideoSourceConfig, +) -> FfiResult { + let codec = config + .codec + .map(|value| { + proto::VideoCodec::try_from(value) + .map(video_codec_from_proto) + .map_err(|_| FfiError::InvalidRequest("invalid codec".into())) + }) + .transpose()?; + + let rate_control = config + .rate_control + .map(|rate_control| { + let unit = proto::GstreamerBitrateUnit::try_from(rate_control.unit) + .map_err(|_| FfiError::InvalidRequest("invalid bitrate unit".into()))?; + Ok::<_, FfiError>(GStreamerRateControlConfig { + element: rate_control.element, + property: rate_control.property, + unit: unit.into(), + }) + }) + .transpose()?; + + Ok(GStreamerVideoSourceConfig { + pipeline: config.pipeline, + codec, + resolution: config.resolution.map(VideoResolution::from), + rate_control, + }) +} diff --git a/livekit-ffi/src/conversion/mod.rs b/livekit-ffi/src/conversion/mod.rs index 364666f24..4ff289823 100644 --- a/livekit-ffi/src/conversion/mod.rs +++ b/livekit-ffi/src/conversion/mod.rs @@ -13,6 +13,8 @@ // limitations under the License. pub mod audio_frame; +#[cfg(feature = "capture")] +pub mod capture; pub mod data_stream; pub mod data_track; pub mod participant; diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs index 4922add55..c984d3828 100644 --- a/livekit-ffi/src/conversion/room.rs +++ b/livekit-ffi/src/conversion/room.rs @@ -63,6 +63,9 @@ fn video_encoder_from_proto(backend: Option) -> Option proto::VideoEncoderBackend::EncoderBackendVideotoolbox => { Some(VideoEncoderBackend::VideoToolbox) } + proto::VideoEncoderBackend::EncoderBackendPreEncoded => { + Some(VideoEncoderBackend::PreEncoded) + } } } diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs new file mode 100644 index 000000000..1264f3627 --- /dev/null +++ b/livekit-ffi/src/server/capture.rs @@ -0,0 +1,390 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! FFI bindings for livekit-capture sources. +//! +//! A capture source owns its producer (e.g. a GStreamer pipeline) and the +//! pump that feeds an RTC video source, so frames never cross the FFI +//! boundary. The RTC source is exposed as a regular [`FfiVideoSource`], so +//! the existing `CreateVideoTrack`/`PublishTrack` requests work unchanged. + +use livekit_capture::{ + encoded::{EncodedVideoPump, EncodedVideoSource}, + pixel::{PixelVideoPump, PixelVideoSource}, + pump::{PumpError, PumpExit, PumpStats, PumpStop, RunningPump}, + sources::{ + clock::ClockVideoSource, + device::{self, DeviceVideoSource}, + gstreamer::GStreamerVideoSource, + pattern::PatternVideoSource, + }, +}; +use parking_lot::Mutex; + +use super::{video_source::FfiVideoSource, FfiHandle, FfiServer}; +use crate::{ + conversion::capture::{ + device_config_from_proto, device_info_to_proto, gstreamer_config_from_proto, + pattern_config_from_proto, video_codec_to_proto, + }, + proto, FfiError, FfiHandleId, FfiResult, +}; + +/// A capture pump of either kind, boxed at the FFI edge. +enum CapturePump { + Pixel(PixelVideoPump>), + Encoded(EncodedVideoPump>), +} + +impl CapturePump { + fn spawn(self) -> std::io::Result { + match self { + Self::Pixel(pump) => pump.spawn(), + Self::Encoded(pump) => pump.spawn(), + } + } +} + +/// State of the capture activity owned by an [`FfiCaptureSource`]. +/// +/// The activity can end without client action (end of stream, error); the +/// FFI object outlives it and is disposed only by the client. +enum CaptureState { + /// Created but not started. + Idle(CapturePump), + /// Started; the watcher task owns the running pump. + Running, + /// The capture ended; the terminal event has been dispatched. + Finished, +} + +pub struct FfiCaptureSource { + pub handle_id: FfiHandleId, + /// Cancellation handle, usable in every state. + stop: PumpStop, + state: Mutex, +} + +impl FfiHandle for FfiCaptureSource {} + +impl Drop for FfiCaptureSource { + fn drop(&mut self) { + // Disposing a running capture stops it; the pump thread observes the + // signal within one bounded wait and drops the source (stopping the + // producer). The watcher task delivers the terminal event, which may + // trail the disposal. + self.stop.stop(); + } +} + +pub fn on_new_capture_source( + server: &'static FfiServer, + request: proto::NewCaptureSourceRequest, +) -> FfiResult { + let async_id = server.resolve_async_id(request.request_async_id); + server.async_runtime.spawn(async move { + let message = match create_capture_source(server, request).await { + Ok(source) => proto::new_capture_source_callback::Message::Source(source), + Err(err) => proto::new_capture_source_callback::Message::Error(err.to_string()), + }; + let _ = server.send_event(proto::ffi_event::Message::NewCaptureSource( + proto::NewCaptureSourceCallback { async_id, message: Some(message) }, + )); + }); + Ok(proto::NewCaptureSourceResponse { async_id }) +} + +async fn create_capture_source( + server: &'static FfiServer, + request: proto::NewCaptureSourceRequest, +) -> FfiResult { + let config = + request.config.ok_or(FfiError::InvalidRequest("missing capture source config".into()))?; + + let pump = match config { + proto::new_capture_source_request::Config::Gstreamer(config) => { + let source = GStreamerVideoSource::new(gstreamer_config_from_proto(config)?) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Encoded(EncodedVideoPump::new(source)) + } + proto::new_capture_source_request::Config::Pattern(config) => { + let source = PatternVideoSource::new(pattern_config_from_proto(config)?) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Pixel(PixelVideoPump::new(source)) + } + proto::new_capture_source_request::Config::Device(config) => { + let source = DeviceVideoSource::new(device_config_from_proto(config)?) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Pixel(PixelVideoPump::new(source)) + } + proto::new_capture_source_request::Config::Clock(config) => { + let source = ClockVideoSource::new(config.into()) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Pixel(PixelVideoPump::new(source)) + } + }; + + let (kind, resolution, codec, publish_options, rtc_source, stop) = match &pump { + CapturePump::Pixel(pump) => ( + proto::CaptureSourceKind::CaptureSourcePixel, + pump.source().resolution(), + None, + pump.publish_options(), + pump.rtc_source(), + pump.stop_handle(), + ), + CapturePump::Encoded(pump) => ( + proto::CaptureSourceKind::CaptureSourceEncoded, + pump.source().resolution(), + Some(pump.source().codec()), + pump.publish_options(), + pump.rtc_source(), + pump.stop_handle(), + ), + }; + + // The RTC source is a regular client-owned handle, used with the + // existing CreateVideoTrack request. + let source_handle_id = server.next_id(); + let video_source = FfiVideoSource { + handle_id: source_handle_id, + source_type: proto::VideoSourceType::VideoSourceNative, + source: rtc_source, + }; + let video_source_info = proto::VideoSourceInfo::from(&video_source); + server.store_handle(source_handle_id, video_source); + + let info = proto::CaptureSourceInfo { + kind: kind.into(), + resolution: proto::VideoSourceResolution { + width: resolution.width, + height: resolution.height, + }, + codec: codec.and_then(video_codec_to_proto).map(Into::into), + recommended_publish_options: recommended_publish_options_to_proto(&publish_options), + video_source: proto::OwnedVideoSource { + handle: proto::FfiOwnedHandle { id: source_handle_id }, + info: video_source_info, + }, + }; + + let capture_handle_id = server.next_id(); + server.store_handle( + capture_handle_id, + FfiCaptureSource { + handle_id: capture_handle_id, + stop, + state: Mutex::new(CaptureState::Idle(pump)), + }, + ); + + Ok(proto::OwnedCaptureSource { handle: proto::FfiOwnedHandle { id: capture_handle_id }, info }) +} + +/// Maps the pump-derived publish options into the proto options the client +/// merges its own settings over. +fn recommended_publish_options_to_proto( + options: &livekit::options::TrackPublishOptions, +) -> proto::TrackPublishOptions { + use livekit::options::VideoCodec; + let video_codec = match options.video_codec { + VideoCodec::VP8 => proto::VideoCodec::Vp8, + VideoCodec::H264 => proto::VideoCodec::H264, + VideoCodec::AV1 => proto::VideoCodec::Av1, + VideoCodec::VP9 => proto::VideoCodec::Vp9, + VideoCodec::H265 => proto::VideoCodec::H265, + }; + let video_encoder = match options.video_encoder { + livekit::options::VideoEncoderBackend::PreEncoded => { + Some(proto::VideoEncoderBackend::EncoderBackendPreEncoded.into()) + } + _ => None, + }; + proto::TrackPublishOptions { + video_codec: Some(video_codec.into()), + video_encoder, + simulcast: Some(options.simulcast), + ..Default::default() + } +} + +pub fn on_start_capture( + server: &'static FfiServer, + request: proto::StartCaptureRequest, +) -> FfiResult { + let capture_handle = request.capture_handle; + let ffi_capture = server.retrieve_handle::(capture_handle)?; + + let mut state = ffi_capture.state.lock(); + let pump = match std::mem::replace(&mut *state, CaptureState::Running) { + CaptureState::Idle(pump) => pump, + other => { + let error = match &other { + CaptureState::Running => "capture is already started", + _ => "capture has already finished", + }; + *state = other; + return Ok(proto::StartCaptureResponse { error: Some(error.to_owned()) }); + } + }; + + let running = match pump.spawn() { + Ok(running) => running, + Err(err) => { + *state = CaptureState::Finished; + return Ok(proto::StartCaptureResponse { + error: Some(format!("failed to start capture: {err}")), + }); + } + }; + drop(state); + drop(ffi_capture); + + // The watcher owns the running pump and delivers the terminal event + // exactly once, whether the capture is stopped, ends, or fails. + server.async_runtime.spawn(async move { + let result = running.join_async().await; + if let Ok(ffi_capture) = server.retrieve_handle::(capture_handle) { + *ffi_capture.state.lock() = CaptureState::Finished; + } + let _ = server.send_event(proto::ffi_event::Message::CaptureSourceEvent( + proto::CaptureSourceEvent { + capture_handle, + message: Some(capture_result_to_proto(result)), + }, + )); + }); + + Ok(proto::StartCaptureResponse { error: None }) +} + +fn capture_result_to_proto( + result: Result, +) -> proto::capture_source_event::Message { + match result { + Ok(stats) => { + let exit = match stats.exit { + PumpExit::Stopped => proto::CaptureExit::Stopped, + PumpExit::EndOfStream => proto::CaptureExit::EndOfStream, + }; + proto::capture_source_event::Message::Finished(proto::CaptureFinished { + frames_captured: stats.frames_captured, + exit: exit.into(), + }) + } + Err(err) => proto::capture_source_event::Message::Error(proto::CaptureError { + error: err.to_string(), + }), + } +} + +pub fn on_stop_capture( + server: &'static FfiServer, + request: proto::StopCaptureRequest, +) -> FfiResult { + let ffi_capture = server.retrieve_handle::(request.capture_handle)?; + ffi_capture.stop.stop(); + Ok(proto::StopCaptureResponse { error: None }) +} + +pub fn on_list_capture_devices( + server: &'static FfiServer, + request: proto::ListCaptureDevicesRequest, +) -> FfiResult { + let async_id = server.resolve_async_id(request.request_async_id); + server.async_runtime.spawn(async move { + let message = match device::devices().await { + Ok(devices) => { + proto::list_capture_devices_callback::Message::Devices(proto::CaptureDeviceList { + devices: devices.into_iter().map(device_info_to_proto).collect(), + }) + } + Err(err) => proto::list_capture_devices_callback::Message::Error(err.to_string()), + }; + let _ = server.send_event(proto::ffi_event::Message::ListCaptureDevices( + proto::ListCaptureDevicesCallback { async_id, message: Some(message) }, + )); + }); + Ok(proto::ListCaptureDevicesResponse { async_id }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::FFI_SERVER; + use std::time::Duration; + + fn server() -> &'static FfiServer { + &FFI_SERVER + } + + #[test] + fn pattern_capture_lifecycle() { + let request = proto::NewCaptureSourceRequest { + config: Some(proto::new_capture_source_request::Config::Pattern( + proto::PatternVideoSourceConfig { + resolution: proto::VideoSourceResolution { width: 1280, height: 720 }, + framerate_fps: 30, + pattern: proto::Pattern::Gradient.into(), + }, + )), + request_async_id: None, + }; + let source = server() + .async_runtime + .block_on(create_capture_source(server(), request)) + .expect("pattern capture source should build"); + assert_eq!(source.info.kind(), proto::CaptureSourceKind::CaptureSourcePixel); + assert_eq!(source.info.resolution.width, 1280); + let capture_handle = source.handle.id; + + // Stopping before starting is allowed; the pump then exits + // immediately once started, and the watcher marks it finished. + let response = + on_stop_capture(server(), proto::StopCaptureRequest { capture_handle }).unwrap(); + assert_eq!(response.error, None); + + let response = + on_start_capture(server(), proto::StartCaptureRequest { capture_handle }).unwrap(); + assert_eq!(response.error, None); + + let response = + on_start_capture(server(), proto::StartCaptureRequest { capture_handle }).unwrap(); + assert!(response.error.is_some(), "double start must be rejected"); + + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + { + let ffi_capture = + server().retrieve_handle::(capture_handle).unwrap(); + if matches!(*ffi_capture.state.lock(), CaptureState::Finished) { + break; + } + } + assert!(std::time::Instant::now() < deadline, "capture did not finish"); + std::thread::sleep(Duration::from_millis(10)); + } + + server().drop_handle(capture_handle); + server().drop_handle(source.info.video_source.handle.id); + } +} diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index a4256789a..df1102f83 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -36,6 +36,8 @@ use crate::{proto, proto::FfiEvent, FfiError, FfiHandleId, FfiResult, INVALID_HA pub mod audio_plugin; pub mod audio_source; pub mod audio_stream; +#[cfg(feature = "capture")] +pub mod capture; pub mod colorcvt; pub mod data_stream; pub mod data_track; diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 8ee569885..5782bb4ff 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -23,6 +23,8 @@ use livekit::{ }; use parking_lot::Mutex; +#[cfg(feature = "capture")] +use super::capture; use super::{ audio_source, audio_stream, colorcvt, data_stream, data_track, participant::FfiParticipant, @@ -1481,6 +1483,24 @@ pub fn handle_request( } Request::StartRecording(req) => platform_audio::on_start_recording(server, req)?.into(), Request::StopRecording(req) => platform_audio::on_stop_recording(server, req)?.into(), + + #[cfg(feature = "capture")] + Request::NewCaptureSource(req) => capture::on_new_capture_source(server, req)?.into(), + #[cfg(feature = "capture")] + Request::StartCapture(req) => capture::on_start_capture(server, req)?.into(), + #[cfg(feature = "capture")] + Request::StopCapture(req) => capture::on_stop_capture(server, req)?.into(), + #[cfg(feature = "capture")] + Request::ListCaptureDevices(req) => capture::on_list_capture_devices(server, req)?.into(), + #[cfg(not(feature = "capture"))] + Request::NewCaptureSource(_) + | Request::StartCapture(_) + | Request::StopCapture(_) + | Request::ListCaptureDevices(_) => { + return Err(FfiError::InvalidRequest( + "livekit-ffi was built without the 'capture' feature".into(), + )); + } }); Ok(res) diff --git a/webrtc-sys/src/video_encoder_factory.cpp b/webrtc-sys/src/video_encoder_factory.cpp index d8b7fd454..e18449c87 100644 --- a/webrtc-sys/src/video_encoder_factory.cpp +++ b/webrtc-sys/src/video_encoder_factory.cpp @@ -17,16 +17,21 @@ #include "livekit/video_encoder_factory.h" #include +#include #include #include #include #include #include +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" + #include "api/environment/environment_factory.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory_template.h" +#include "livekit/encoded_video_frame_buffer.h" #include "livekit/objc_video_factory.h" #include "livekit/passthrough_video_encoder.h" #include "livekit/webrtc.h" @@ -605,6 +610,80 @@ VideoEncoderFactory::CodecSupport VideoEncoderFactory::QueryCodecSupport( return internal_factory_->QueryCodecSupport(format, scalability_mode); } +namespace { + +// Real encoders can never consume pre-encoded access units, but frames +// carrying an EncodedVideoFrameBuffer can still reach one in the window +// between stream startup and the sender's encoder selector switching onto +// the pass-through backend. Some platform encoders blind-cast native +// buffers (macOS ObjCVideoEncoder casts to ObjCFrameBuffer and retains a +// garbage pointer), so forwarding such a frame is a crash, not a graceful +// failure. Drop it instead: the selector switches shortly after, and the +// pass-through encoder requests a fresh keyframe when it takes over. +class EncodedFrameGuardEncoder final : public webrtc::VideoEncoder { + public: + explicit EncodedFrameGuardEncoder( + std::unique_ptr encoder) + : encoder_(std::move(encoder)) {} + + void SetFecControllerOverride( + webrtc::FecControllerOverride* fec_controller_override) override { + encoder_->SetFecControllerOverride(fec_controller_override); + } + + int InitEncode(const webrtc::VideoCodec* codec_settings, + const Settings& settings) override { + return encoder_->InitEncode(codec_settings, settings); + } + + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) override { + return encoder_->RegisterEncodeCompleteCallback(callback); + } + + int32_t Release() override { return encoder_->Release(); } + + int32_t Encode( + const webrtc::VideoFrame& frame, + const std::vector* frame_types) override { + if (livekit::EncodedVideoFrameBuffer::FromNative( + frame.video_frame_buffer().get())) { + static std::atomic logged{false}; + if (!logged.exchange(true)) { + RTC_LOG(LS_WARNING) + << "Dropping pre-encoded access unit sent to a non pass-through " + "encoder; waiting for the sender to switch onto the " + "pass-through backend"; + } + return WEBRTC_VIDEO_CODEC_OK; + } + return encoder_->Encode(frame, frame_types); + } + + void SetRates(const RateControlParameters& parameters) override { + encoder_->SetRates(parameters); + } + + void OnPacketLossRateUpdate(float packet_loss_rate) override { + encoder_->OnPacketLossRateUpdate(packet_loss_rate); + } + + void OnRttUpdate(int64_t rtt_ms) override { encoder_->OnRttUpdate(rtt_ms); } + + void OnLossNotification(const LossNotification& loss_notification) override { + encoder_->OnLossNotification(loss_notification); + } + + EncoderInfo GetEncoderInfo() const override { + return encoder_->GetEncoderInfo(); + } + + private: + std::unique_ptr encoder_; +}; + +} // namespace + std::unique_ptr VideoEncoderFactory::Create( const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) { @@ -614,6 +693,11 @@ std::unique_ptr VideoEncoderFactory::Create( env, internal_factory_.get(), nullptr, format); } + if (encoder && + BackendFromFormat(format) != VideoEncoderBackend::PreEncoded) { + encoder = std::make_unique(std::move(encoder)); + } + return encoder; }