diff --git a/apps/skit/src/server.rs b/apps/skit/src/server.rs index ba151091..987a493b 100644 --- a/apps/skit/src/server.rs +++ b/apps/skit/src/server.rs @@ -1520,6 +1520,12 @@ struct CreateSessionResponse { async fn populate_session_pipeline(session: &crate::session::Session, engine_pipeline: &Pipeline) { let mut pipeline = session.pipeline.lock().await; + // Forward top-level metadata so the UI can read it from the session snapshot. + pipeline.name.clone_from(&engine_pipeline.name); + pipeline.description.clone_from(&engine_pipeline.description); + pipeline.mode = engine_pipeline.mode; + pipeline.client.clone_from(&engine_pipeline.client); + // Add nodes to in-memory pipeline for (node_id, node_spec) in &engine_pipeline.nodes { pipeline.nodes.insert( diff --git a/apps/skit/src/websocket_handlers.rs b/apps/skit/src/websocket_handlers.rs index 96cef940..248a448b 100644 --- a/apps/skit/src/websocket_handlers.rs +++ b/apps/skit/src/websocket_handlers.rs @@ -1105,7 +1105,7 @@ async fn handle_get_pipeline( "Retrieved pipeline" ); - Some(ResponsePayload::Pipeline { pipeline: api_pipeline }) + Some(ResponsePayload::Pipeline { pipeline: Box::new(api_pipeline) }) } fn handle_validate_batch( diff --git a/crates/api/src/bin/generate_ts_types.rs b/crates/api/src/bin/generate_ts_types.rs index 02943730..7354fdbe 100644 --- a/crates/api/src/bin/generate_ts_types.rs +++ b/crates/api/src/bin/generate_ts_types.rs @@ -63,6 +63,16 @@ fn main() -> Result<(), Box> { format!("export {}", streamkit_api::ValidationError::decl(&cfg)), format!("export {}", streamkit_api::ValidationErrorType::decl(&cfg)), format!("export {}", streamkit_api::PermissionsInfo::decl(&cfg)), + // Declarative client section types + format!("\n// client section\nexport {}", streamkit_api::yaml::ClientSection::decl(&cfg)), + format!("export {}", streamkit_api::yaml::PublishConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::WatchConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::InputConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::InputType::decl(&cfg)), + format!("export {}", streamkit_api::yaml::OutputConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::OutputType::decl(&cfg)), + format!("export {}", streamkit_api::yaml::FieldHint::decl(&cfg)), + format!("export {}", streamkit_api::yaml::FieldType::decl(&cfg)), ]; let output = declarations.join("\n\n"); diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 07af1903..67285cbf 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -328,7 +328,7 @@ pub enum ResponsePayload { nodes: Vec, }, Pipeline { - pipeline: ApiPipeline, + pipeline: Box, }, ValidationResult { errors: Vec, @@ -538,6 +538,11 @@ pub struct Pipeline { pub description: Option, #[serde(default)] pub mode: EngineMode, + /// Declarative UI metadata — forwarded unchanged from `UserPipeline`, + /// ignored by the engine for execution. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub client: Option, #[ts(type = "Record")] pub nodes: indexmap::IndexMap, pub connections: Vec, diff --git a/crates/api/src/yaml.rs b/crates/api/src/yaml.rs index 86500178..e9b02ff9 100644 --- a/crates/api/src/yaml.rs +++ b/crates/api/src/yaml.rs @@ -11,7 +11,8 @@ use super::{Connection, ConnectionMode, EngineMode, Node, Pipeline}; use indexmap::IndexMap; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; /// Represents a single step in a linear pipeline definition. #[derive(Debug, Deserialize)] @@ -92,6 +93,146 @@ pub enum Needs { Map(IndexMap), } +// --------------------------------------------------------------------------- +// Declarative `client` section — UI metadata for pipeline rendering +// --------------------------------------------------------------------------- + +/// Top-level `client` section in pipeline YAML. +/// +/// Declares what the browser UI should do when rendering this pipeline. +/// Dynamic pipelines use `relay_url`/`gateway_path`/`publish`/`watch`; +/// oneshot pipelines use `input`/`output`. The two sets are mutually +/// exclusive by mode (enforced by the lint pass, not at parse time). +#[derive(Debug, Clone, Default, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct ClientSection { + /// Direct relay URL for external MoQ relay pattern. + pub relay_url: Option, + /// Gateway path for gateway-managed MoQ pattern. + pub gateway_path: Option, + /// Browser-side publish configuration (dynamic pipelines). + pub publish: Option, + /// Browser-side watch configuration (dynamic pipelines). + pub watch: Option, + /// Input UX configuration (oneshot pipelines). + pub input: Option, + /// Output rendering configuration (oneshot pipelines). + pub output: Option, +} + +/// Browser-side publish configuration for dynamic pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct PublishConfig { + /// Broadcast name the browser publishes to. + pub broadcast: String, + /// Whether the pipeline consumes audio from the browser. + #[serde(default)] + pub audio: bool, + /// Whether the pipeline consumes video from the browser. + #[serde(default)] + pub video: bool, +} + +/// Browser-side watch configuration for dynamic pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct WatchConfig { + /// Broadcast name the browser subscribes to. + pub broadcast: String, + /// Whether the pipeline outputs audio to subscribers. + #[serde(default)] + pub audio: bool, + /// Whether the pipeline outputs video to subscribers. + #[serde(default)] + pub video: bool, +} + +/// Input UX configuration for oneshot pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct InputConfig { + /// The kind of input UX to present. + #[serde(rename = "type")] + pub input_type: InputType, + /// MIME filter for file pickers (e.g. `audio/*`). + pub accept: Option, + /// Tags for filtering the asset picker (e.g. `["speech"]`). + pub asset_tags: Option>, + /// Placeholder text for text inputs. + pub placeholder: Option, + /// Per-field UI hints keyed by `http_input` field name. + #[ts(type = "Record | null")] + pub field_hints: Option>, +} + +/// The kind of input UX a oneshot pipeline expects. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS)] +#[ts(export)] +#[serde(rename_all = "snake_case")] +pub enum InputType { + /// File upload with optional MIME filter. + FileUpload, + /// Free-form text input. + Text, + /// Has `http_input` but the body is irrelevant (trigger only). + Trigger, + /// No `http_input` node — pipeline generates its own input. + None, +} + +/// Output rendering configuration for oneshot pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct OutputConfig { + /// The media kind the pipeline produces. + #[serde(rename = "type")] + pub output_type: OutputType, +} + +/// The media kind a oneshot pipeline produces. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS)] +#[ts(export)] +#[serde(rename_all = "snake_case")] +pub enum OutputType { + /// Transcription segments (JSON stream). + Transcription, + /// Generic JSON stream. + Json, + /// Audio output. + Audio, + /// Video output. + Video, +} + +/// Per-field input type discriminator for `field_hints`. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS)] +#[ts(export)] +#[serde(rename_all = "snake_case")] +pub enum FieldType { + /// File upload field. + File, + /// Text input field. + Text, +} + +/// Per-field UI hint within `InputConfig.field_hints`. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct FieldHint { + /// Override the field's input type (default is file upload). + #[serde(rename = "type")] + pub field_type: Option, + /// MIME filter for file picker. + pub accept: Option, + /// Placeholder text for text inputs. + pub placeholder: Option, +} + +// --------------------------------------------------------------------------- +// User-facing pipeline definition +// --------------------------------------------------------------------------- + /// The top-level structure for a user-facing pipeline definition. /// `serde(untagged)` allows it to be parsed as either a steps-based /// pipeline or a nodes-based (DAG) pipeline. @@ -106,6 +247,8 @@ pub enum UserPipeline { #[serde(default)] mode: EngineMode, steps: Vec, + /// Declarative UI metadata (optional — required only for UI rendering). + client: Option, }, Dag { #[serde(skip_serializing_if = "Option::is_none")] @@ -115,6 +258,8 @@ pub enum UserPipeline { #[serde(default)] mode: EngineMode, nodes: IndexMap, + /// Declarative UI metadata (optional — required only for UI rendering). + client: Option, }, } @@ -131,6 +276,15 @@ pub enum UserPipeline { pub fn parse_yaml(yaml: &str) -> Result { let json_value: serde_json::Value = serde_saphyr::from_str(yaml).map_err(|e| format!("Invalid YAML: {e}"))?; + + // Pre-validate `client` if present so that enum/type errors produce + // actionable messages instead of collapsing into the generic + // "did not match any variant" error from the untagged `UserPipeline`. + if let Some(client_val) = json_value.get("client") { + let _: ClientSection = serde_json::from_value(client_val.clone()) + .map_err(|e| format!("Invalid client section: {e}"))?; + } + serde_json::from_value(json_value).map_err(|e| format!("Invalid pipeline: {e}")) } @@ -141,11 +295,11 @@ pub fn parse_yaml(yaml: &str) -> Result { /// Returns an error if a node references a non-existent dependency in its `needs` field. pub fn compile(pipeline: UserPipeline) -> Result { match pipeline { - UserPipeline::Steps { name, description, mode, steps } => { - Ok(compile_steps(name, description, mode, steps)) + UserPipeline::Steps { name, description, mode, steps, client } => { + Ok(compile_steps(name, description, mode, steps, client)) }, - UserPipeline::Dag { name, description, mode, nodes } => { - compile_dag(name, description, mode, nodes) + UserPipeline::Dag { name, description, mode, nodes, client } => { + compile_dag(name, description, mode, nodes, client) }, } } @@ -156,6 +310,7 @@ fn compile_steps( description: Option, mode: EngineMode, steps: Vec, + client: Option, ) -> Pipeline { let mut nodes = IndexMap::new(); let mut connections = Vec::new(); @@ -177,7 +332,7 @@ fn compile_steps( nodes.insert(node_name, Node { kind: step.kind, params: step.params, state: None }); } - Pipeline { name, description, mode, nodes, connections, view_data: None } + Pipeline { name, description, mode, client, nodes, connections, view_data: None } } /// Known bidirectional node kinds that are allowed to participate in cycles. @@ -300,6 +455,7 @@ fn compile_dag( description: Option, mode: EngineMode, user_nodes: IndexMap, + client: Option, ) -> Result { // First, detect cycles in the dependency graph detect_cycles(&user_nodes)?; @@ -409,7 +565,459 @@ fn compile_dag( }) .collect(); - Ok(Pipeline { name, description, mode, nodes, connections, view_data: None }) + Ok(Pipeline { name, description, mode, client, nodes, connections, view_data: None }) +} + +// --------------------------------------------------------------------------- +// Client section lint pass — semantic validation +// --------------------------------------------------------------------------- + +/// A single lint warning produced by [`lint_client_section`]. +#[derive(Debug, Clone)] +pub struct ClientLintWarning { + /// Machine-readable rule identifier (e.g. `"mode-mismatch"`). + pub rule: &'static str, + /// Human-readable description of the problem. + pub message: String, +} + +/// Validates the `client` section against the compiled pipeline, returning +/// any semantic warnings. +/// +/// This is a *lint* pass — it never prevents compilation, but surfaces +/// likely authoring mistakes so tooling (CLI, editor integrations) can +/// flag them. +/// +/// # Rules +/// +/// 1. **`mode-mismatch-dynamic`** — Dynamic pipeline declares oneshot-only +/// fields (`input` / `output`). +/// 2. **`mode-mismatch-oneshot`** — Oneshot pipeline declares dynamic-only +/// fields (`publish` / `watch` / `gateway_path` / `relay_url`). +/// 3. **`missing-gateway`** — Dynamic pipeline has `publish` or `watch` +/// but no `gateway_path` or `relay_url`. +/// 4. **`publish-no-media`** — `publish` block sets both `audio` and +/// `video` to false. +/// 5. **`watch-no-media`** — `watch` block sets both `audio` and `video` +/// to false. +/// 6. **`input-none-with-accept`** — `input.type` is `none` but `accept` +/// is set (accept is meaningless without a file picker). +/// 7. **`input-trigger-with-accept`** — `input.type` is `trigger` but +/// `accept` is set. +/// 8. **`field-hints-no-input`** — `field_hints` is present but +/// `input.type` is `none`. +/// 9. **`asset-tags-no-input`** — `asset_tags` is present but +/// `input.type` is `none` or `text`. +/// 10. **`text-no-placeholder`** — `input.type` is `text` but no +/// `placeholder` is provided (best-practice hint). +/// 11. **`empty-broadcast`** — `publish.broadcast` or `watch.broadcast` +/// is an empty string. +/// 12. **`duplicate-broadcast`** — `publish.broadcast` equals +/// `watch.broadcast` (would cause a loop). +pub fn lint_client_section(client: &ClientSection, mode: EngineMode) -> Vec { + let mut warnings = Vec::new(); + + let has_dynamic_fields = client.gateway_path.is_some() + || client.relay_url.is_some() + || client.publish.is_some() + || client.watch.is_some(); + + let has_oneshot_fields = client.input.is_some() || client.output.is_some(); + + // Rule 1: dynamic pipeline with oneshot-only fields + if mode == EngineMode::Dynamic && has_oneshot_fields { + warnings.push(ClientLintWarning { + rule: "mode-mismatch-dynamic", + message: "Dynamic pipeline declares `input` or `output` — these are oneshot-only \ + fields and will be ignored." + .into(), + }); + } + + // Rule 2: oneshot pipeline with dynamic-only fields + if mode == EngineMode::OneShot && has_dynamic_fields { + warnings.push(ClientLintWarning { + rule: "mode-mismatch-oneshot", + message: "Oneshot pipeline declares `publish`, `watch`, `gateway_path`, or \ + `relay_url` — these are dynamic-only fields and will be ignored." + .into(), + }); + } + + // Rule 3: missing gateway + if (client.publish.is_some() || client.watch.is_some()) + && client.gateway_path.is_none() + && client.relay_url.is_none() + { + warnings.push(ClientLintWarning { + rule: "missing-gateway", + message: "Pipeline has `publish` or `watch` but no `gateway_path` or `relay_url` — \ + the browser won't know where to connect." + .into(), + }); + } + + // Rule 4: publish with no media + if let Some(ref publish) = client.publish { + if !publish.audio && !publish.video { + warnings.push(ClientLintWarning { + rule: "publish-no-media", + message: "publish block sets both `audio` and `video` to false — nothing will be \ + sent from the browser." + .into(), + }); + } + + // Rule 11a: empty broadcast + if publish.broadcast.is_empty() { + warnings.push(ClientLintWarning { + rule: "empty-broadcast", + message: "publish.broadcast is an empty string.".into(), + }); + } + } + + // Rule 5: watch with no media + if let Some(ref watch) = client.watch { + if !watch.audio && !watch.video { + warnings.push(ClientLintWarning { + rule: "watch-no-media", + message: "watch block sets both `audio` and `video` to false — nothing will be \ + received by the browser." + .into(), + }); + } + + // Rule 11b: empty broadcast + if watch.broadcast.is_empty() { + warnings.push(ClientLintWarning { + rule: "empty-broadcast", + message: "watch.broadcast is an empty string.".into(), + }); + } + } + + // Rule 12: duplicate broadcast + if let (Some(ref publish), Some(ref watch)) = (&client.publish, &client.watch) { + if !publish.broadcast.is_empty() && publish.broadcast == watch.broadcast { + warnings.push(ClientLintWarning { + rule: "duplicate-broadcast", + message: format!( + "publish.broadcast and watch.broadcast are both '{}' — this would \ + cause a feedback loop.", + publish.broadcast + ), + }); + } + } + + // Input-related rules + if let Some(ref input) = client.input { + // Rule 6: input none with accept + if matches!(input.input_type, InputType::None) && input.accept.is_some() { + warnings.push(ClientLintWarning { + rule: "input-none-with-accept", + message: "input.type is `none` but `accept` is set — accept is meaningless \ + without a file picker." + .into(), + }); + } + + // Rule 7: input trigger with accept + if matches!(input.input_type, InputType::Trigger) && input.accept.is_some() { + warnings.push(ClientLintWarning { + rule: "input-trigger-with-accept", + message: "input.type is `trigger` but `accept` is set — accept is meaningless \ + for trigger inputs." + .into(), + }); + } + + // Rule 8: field_hints with no input + if matches!(input.input_type, InputType::None) + && input.field_hints.as_ref().is_some_and(|h| !h.is_empty()) + { + warnings.push(ClientLintWarning { + rule: "field-hints-no-input", + message: "field_hints is present but input.type is `none` — hints are unused \ + without an input." + .into(), + }); + } + + // Rule 9: asset_tags with no input or text input + if matches!(input.input_type, InputType::None | InputType::Text) + && input.asset_tags.as_ref().is_some_and(|t| !t.is_empty()) + { + warnings.push(ClientLintWarning { + rule: "asset-tags-no-input", + message: "asset_tags is present but input.type is `none` or `text` — tags are \ + only useful for file_upload inputs." + .into(), + }); + } + + // Rule 10: text input without placeholder + if matches!(input.input_type, InputType::Text) && input.placeholder.is_none() { + warnings.push(ClientLintWarning { + rule: "text-no-placeholder", + message: "input.type is `text` but no `placeholder` is provided — consider \ + adding one for a better UX." + .into(), + }); + } + } + + warnings +} + +/// A lightweight view of a pipeline's nodes used by +/// [`lint_client_against_nodes`] for cross-validation. +/// +/// Callers construct this from either `UserPipeline::Dag` nodes or +/// `UserPipeline::Steps` steps. +pub struct NodeInfo<'a> { + pub kind: &'a str, + pub params: Option<&'a serde_json::Value>, +} + +/// Cross-validates the `client` section against the pipeline's node graph. +/// +/// This is a second lint layer that complements [`lint_client_section`] +/// (which checks `client` in isolation). The rules here require knowledge +/// of which nodes exist and their params. +/// +/// # Rules +/// +/// 13. **`input-requires-http-input`** — `input.type` is `file_upload`, +/// `text`, or `trigger` but no `streamkit::http_input` node exists. +/// 14. **`input-none-has-http-input`** — `input.type` is `none` but an +/// `streamkit::http_input` node exists (should be `trigger`). +/// 15. **`field-hint-unknown-field`** — `field_hints` references a field +/// name not found in any `streamkit::http_input` node's `fields` param. +/// 16. **`publish-no-transport`** — `publish` is declared but no MoQ +/// transport node (`transport::moq::peer` or +/// `transport::moq::subscriber`) exists. +/// 17. **`watch-no-transport`** — `watch` is declared but no MoQ transport +/// node (`transport::moq::peer` or `transport::moq::publisher`) exists. +/// 18. **`gateway-path-mismatch`** — `client.gateway_path` does not match +/// the `gateway_path` param on a `transport::moq::peer` node. +/// 19. **`relay-url-mismatch`** — `client.relay_url` does not match the +/// `url` param on a `transport::moq::publisher` or +/// `transport::moq::subscriber` node. +/// 20. **`broadcast-mismatch`** — `publish.broadcast` or `watch.broadcast` +/// does not match any broadcast name configured on MoQ transport nodes. +pub fn lint_client_against_nodes( + client: &ClientSection, + _mode: EngineMode, + nodes: &[NodeInfo<'_>], +) -> Vec { + let mut warnings = Vec::new(); + + // Collect node kinds and params for efficient lookup. + let has_http_input = nodes.iter().any(|n| n.kind == "streamkit::http_input"); + let has_moq_peer = nodes.iter().any(|n| n.kind == "transport::moq::peer"); + let has_moq_subscriber = nodes.iter().any(|n| n.kind == "transport::moq::subscriber"); + let has_moq_publisher = nodes.iter().any(|n| n.kind == "transport::moq::publisher"); + + // Rule 13: input requires http_input node + if let Some(ref input) = client.input { + let needs_http_input = matches!( + input.input_type, + InputType::FileUpload | InputType::Text | InputType::Trigger + ); + if needs_http_input && !has_http_input { + warnings.push(ClientLintWarning { + rule: "input-requires-http-input", + message: format!( + "input.type is `{}` but no `streamkit::http_input` node exists.", + match input.input_type { + InputType::FileUpload => "file_upload", + InputType::Text => "text", + InputType::Trigger => "trigger", + InputType::None => "none", + } + ), + }); + } + + // Rule 14: input.type is none but http_input exists + if matches!(input.input_type, InputType::None) && has_http_input { + warnings.push(ClientLintWarning { + rule: "input-none-has-http-input", + message: "input.type is `none` but a `streamkit::http_input` node exists — \ + consider using `trigger` instead." + .into(), + }); + } + + // Rule 15: field_hints references unknown field names + if let Some(ref hints) = input.field_hints { + let mut declared_fields: Vec = Vec::new(); + for node in nodes.iter().filter(|n| n.kind == "streamkit::http_input") { + if let Some(params) = node.params { + // Single field: { field: { name: "foo" } } + if let Some(name) = + params.get("field").and_then(|f| f.get("name")).and_then(|n| n.as_str()) + { + declared_fields.push(name.to_string()); + } + // Multi field: { fields: [{ name: "foo" }, { name: "bar" }] } + if let Some(fields_arr) = params.get("fields").and_then(|f| f.as_array()) { + for f in fields_arr { + if let Some(name) = f.get("name").and_then(|n| n.as_str()) { + declared_fields.push(name.to_string()); + } + } + } + } + // http_input with no params has a single default field named "media" + if (node.params.is_none() + || node + .params + .is_none_or(|p| p.get("field").is_none() && p.get("fields").is_none())) + && !declared_fields.contains(&"media".to_string()) + { + declared_fields.push("media".to_string()); + } + } + + if !declared_fields.is_empty() { + for hint_name in hints.keys() { + if !declared_fields.iter().any(|f| f == hint_name) { + warnings.push(ClientLintWarning { + rule: "field-hint-unknown-field", + message: format!( + "field_hints references `{hint_name}` but no `streamkit::http_input` \ + node declares a field with that name. Known fields: {}.", + declared_fields.join(", ") + ), + }); + } + } + } + } + } + + // Rule 16: publish but no MoQ subscriber/peer + // Browser publish = server subscribes → need moq::peer or moq::subscriber + if client.publish.is_some() && !has_moq_peer && !has_moq_subscriber { + warnings.push(ClientLintWarning { + rule: "publish-no-transport", + message: "client declares `publish` but no `transport::moq::peer` or \ + `transport::moq::subscriber` node exists." + .into(), + }); + } + + // Rule 17: watch but no MoQ publisher/peer + // Browser watch = server publishes → need moq::peer or moq::publisher + if client.watch.is_some() && !has_moq_peer && !has_moq_publisher { + warnings.push(ClientLintWarning { + rule: "watch-no-transport", + message: "client declares `watch` but no `transport::moq::peer` or \ + `transport::moq::publisher` node exists." + .into(), + }); + } + + // Rule 18: gateway_path mismatch with moq::peer node + if let Some(ref client_gw) = client.gateway_path { + let peer_gateway_paths: Vec<&str> = nodes + .iter() + .filter(|n| n.kind == "transport::moq::peer") + .filter_map(|n| n.params.and_then(|p| p.get("gateway_path")).and_then(|v| v.as_str())) + .collect(); + + if !peer_gateway_paths.is_empty() && !peer_gateway_paths.iter().any(|gw| gw == client_gw) { + warnings.push(ClientLintWarning { + rule: "gateway-path-mismatch", + message: format!( + "client.gateway_path is `{client_gw}` but moq::peer node(s) declare: {}.", + peer_gateway_paths.join(", ") + ), + }); + } + } + + // Rule 19: relay_url mismatch with publisher/subscriber nodes + if let Some(ref client_url) = client.relay_url { + let node_urls: Vec<&str> = nodes + .iter() + .filter(|n| { + n.kind == "transport::moq::publisher" || n.kind == "transport::moq::subscriber" + }) + .filter_map(|n| n.params.and_then(|p| p.get("url")).and_then(|v| v.as_str())) + .collect(); + + if !node_urls.is_empty() && !node_urls.iter().any(|u| u == client_url) { + warnings.push(ClientLintWarning { + rule: "relay-url-mismatch", + message: format!( + "client.relay_url is `{client_url}` but transport node(s) declare: {}.", + node_urls.join(", ") + ), + }); + } + } + + // Rule 20: broadcast name mismatch + // Collect all broadcast names from MoQ transport nodes. + let mut node_broadcasts: Vec<&str> = Vec::new(); + for node in nodes { + if let Some(params) = node.params { + match node.kind { + "transport::moq::peer" => { + if let Some(b) = params.get("input_broadcast").and_then(|v| v.as_str()) { + node_broadcasts.push(b); + } + if let Some(b) = params.get("output_broadcast").and_then(|v| v.as_str()) { + node_broadcasts.push(b); + } + }, + "transport::moq::publisher" | "transport::moq::subscriber" => { + if let Some(b) = params.get("broadcast").and_then(|v| v.as_str()) { + node_broadcasts.push(b); + } + }, + _ => {}, + } + } + } + + if !node_broadcasts.is_empty() { + if let Some(ref publish) = client.publish { + if !publish.broadcast.is_empty() + && !node_broadcasts.iter().any(|b| *b == publish.broadcast) + { + warnings.push(ClientLintWarning { + rule: "broadcast-mismatch", + message: format!( + "publish.broadcast is `{}` but no MoQ transport node declares \ + that broadcast name. Node broadcasts: {}.", + publish.broadcast, + node_broadcasts.join(", ") + ), + }); + } + } + if let Some(ref watch) = client.watch { + if !watch.broadcast.is_empty() && !node_broadcasts.iter().any(|b| *b == watch.broadcast) + { + warnings.push(ClientLintWarning { + rule: "broadcast-mismatch", + message: format!( + "watch.broadcast is `{}` but no MoQ transport node declares \ + that broadcast name. Node broadcasts: {}.", + watch.broadcast, + node_broadcasts.join(", ") + ), + }); + } + } + } + + warnings } #[cfg(test)] @@ -968,4 +1576,718 @@ nodes: assert_eq!(conn.to_node, "sink"); assert_eq!(conn.to_pin, "my_input"); } + + // ----------------------------------------------------------------------- + // Client section tests + // ----------------------------------------------------------------------- + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_parsed_in_steps() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input + - kind: streamkit::http_output +client: + input: + type: file_upload + accept: "audio/*" + output: + type: transcription +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + let input = client.input.expect("input config should be present"); + assert!(matches!(input.input_type, InputType::FileUpload)); + assert_eq!(input.accept.as_deref(), Some("audio/*")); + + let output = client.output.expect("output config should be present"); + assert!(matches!(output.output_type, OutputType::Transcription)); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_parsed_in_dag() { + let yaml = r#" +mode: dynamic +nodes: + peer: + kind: transport::moq::peer + params: + gateway_path: /moq/test + input_broadcast: camera + output_broadcast: output +client: + gateway_path: /moq/test + publish: + broadcast: camera + audio: true + video: true + watch: + broadcast: output + audio: true + video: true +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + assert_eq!(client.gateway_path.as_deref(), Some("/moq/test")); + + let publish = client.publish.expect("publish config should be present"); + assert_eq!(publish.broadcast, "camera"); + assert!(publish.audio); + assert!(publish.video); + + let watch = client.watch.expect("watch config should be present"); + assert_eq!(watch.broadcast, "output"); + assert!(watch.audio); + assert!(watch.video); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_optional() { + let yaml = r" +mode: oneshot +steps: + - kind: core::passthrough +"; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + assert!(compiled.client.is_none()); + } + + #[test] + fn test_invalid_client_section_rejected() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input +client: + input: + type: invalid_type +"#; + let result = parse_yaml(yaml); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.contains("Invalid client section"), + "Error should mention client section: {err}" + ); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_with_field_hints() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input + - kind: streamkit::http_output +client: + input: + type: file_upload + accept: "audio/*" + field_hints: + text: + type: text + placeholder: "Enter your prompt" + reference: + type: file + accept: "audio/*" + output: + type: audio +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + let input = client.input.expect("input config should be present"); + let hints = input.field_hints.expect("field_hints should be present"); + + assert_eq!(hints.len(), 2); + + let text_hint = hints.get("text").expect("text hint should exist"); + assert!(matches!(text_hint.field_type, Some(FieldType::Text))); + assert_eq!(text_hint.placeholder.as_deref(), Some("Enter your prompt")); + + let ref_hint = hints.get("reference").expect("reference hint should exist"); + assert!(matches!(ref_hint.field_type, Some(FieldType::File))); + assert_eq!(ref_hint.accept.as_deref(), Some("audio/*")); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_with_asset_tags() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input + - kind: streamkit::http_output +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + - voice + output: + type: transcription +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + let input = client.input.expect("input config should be present"); + let tags = input.asset_tags.expect("asset_tags should be present"); + assert_eq!(tags, vec!["speech", "voice"]); + } + + // ----------------------------------------------------------------------- + // Client section lint tests + // ----------------------------------------------------------------------- + + /// Helper to build a minimal valid dynamic client section. + fn dynamic_client() -> ClientSection { + ClientSection { + relay_url: None, + gateway_path: Some("/moq/test".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + watch: Some(WatchConfig { broadcast: "output".into(), audio: true, video: true }), + input: None, + output: None, + } + } + + /// Helper to build a minimal valid oneshot client section. + fn oneshot_client() -> ClientSection { + ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + } + } + + #[test] + fn test_lint_clean_dynamic() { + let warnings = lint_client_section(&dynamic_client(), EngineMode::Dynamic); + assert!(warnings.is_empty(), "Expected no warnings: {warnings:?}"); + } + + #[test] + fn test_lint_clean_oneshot() { + let warnings = lint_client_section(&oneshot_client(), EngineMode::OneShot); + assert!(warnings.is_empty(), "Expected no warnings: {warnings:?}"); + } + + #[test] + fn test_lint_mode_mismatch_dynamic_with_oneshot_fields() { + let mut c = dynamic_client(); + c.input = Some(InputConfig { + input_type: InputType::FileUpload, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: None, + }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "mode-mismatch-dynamic")); + } + + #[test] + fn test_lint_mode_mismatch_oneshot_with_dynamic_fields() { + let mut c = oneshot_client(); + c.gateway_path = Some("/moq/test".into()); + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "mode-mismatch-oneshot")); + } + + #[test] + fn test_lint_missing_gateway() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: Some(PublishConfig { broadcast: "x".into(), audio: true, video: false }), + watch: None, + input: None, + output: None, + }; + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "missing-gateway")); + } + + #[test] + fn test_lint_publish_no_media() { + let mut c = dynamic_client(); + c.publish = Some(PublishConfig { broadcast: "x".into(), audio: false, video: false }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "publish-no-media")); + } + + #[test] + fn test_lint_watch_no_media() { + let mut c = dynamic_client(); + c.watch = Some(WatchConfig { broadcast: "x".into(), audio: false, video: false }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "watch-no-media")); + } + + #[test] + fn test_lint_empty_broadcast() { + let mut c = dynamic_client(); + c.publish = Some(PublishConfig { broadcast: String::new(), audio: true, video: false }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "empty-broadcast")); + } + + #[test] + fn test_lint_duplicate_broadcast() { + let mut c = dynamic_client(); + c.publish = Some(PublishConfig { broadcast: "same".into(), audio: true, video: false }); + c.watch = Some(WatchConfig { broadcast: "same".into(), audio: true, video: true }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "duplicate-broadcast")); + } + + #[test] + fn test_lint_input_none_with_accept() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::None, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Video }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "input-none-with-accept")); + } + + #[test] + fn test_lint_input_trigger_with_accept() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::Trigger, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "input-trigger-with-accept")); + } + + #[test] + fn test_lint_field_hints_no_input() { + let mut hints = IndexMap::new(); + hints.insert( + "x".into(), + FieldHint { field_type: Some(FieldType::File), accept: None, placeholder: None }, + ); + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::None, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Video }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "field-hints-no-input")); + } + + #[test] + fn test_lint_asset_tags_text_input() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::Text, + accept: None, + asset_tags: Some(vec!["speech".into()]), + placeholder: Some("Enter text".into()), + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "asset-tags-no-input")); + } + + #[test] + fn test_lint_text_no_placeholder() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::Text, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "text-no-placeholder")); + } + + // ----------------------------------------------------------------------- + // Client-vs-nodes cross-validation tests (rules 13–20) + // ----------------------------------------------------------------------- + + /// Helper: a `streamkit::http_input` node with no params. + fn http_input_node() -> serde_json::Value { + serde_json::Value::Null // represents "no params object" + } + + fn node<'a>(kind: &'a str, params: Option<&'a serde_json::Value>) -> NodeInfo<'a> { + NodeInfo { kind, params } + } + + // Rule 13 — input-requires-http-input + #[test] + fn test_lint_input_requires_http_input() { + let c = oneshot_client(); // input.type = file_upload + let nodes: Vec> = vec![]; // no http_input + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!(warnings.iter().any(|w| w.rule == "input-requires-http-input")); + } + + #[test] + fn test_lint_input_requires_http_input_clean() { + let c = oneshot_client(); + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "input-requires-http-input"), + "Should not warn when http_input exists: {warnings:?}" + ); + } + + // Rule 14 — input-none-has-http-input + #[test] + fn test_lint_input_none_has_http_input() { + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::None, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Video }), + ..Default::default() + }; + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!(warnings.iter().any(|w| w.rule == "input-none-has-http-input")); + } + + // Rule 15 — field-hint-unknown-field + #[test] + fn test_lint_field_hint_unknown_field() { + let mut hints = IndexMap::new(); + hints.insert( + "nonexistent".into(), + FieldHint { field_type: Some(FieldType::Text), accept: None, placeholder: None }, + ); + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + ..Default::default() + }; + // http_input with no explicit field/fields → default field is "media" + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "field-hint-unknown-field"), + "Should warn for unknown field hint name: {warnings:?}" + ); + } + + #[test] + fn test_lint_field_hint_known_field_clean() { + let mut hints = IndexMap::new(); + hints.insert( + "media".into(), + FieldHint { + field_type: Some(FieldType::File), + accept: Some("audio/*".into()), + placeholder: None, + }, + ); + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + ..Default::default() + }; + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "field-hint-unknown-field"), + "Should not warn for default 'media' field: {warnings:?}" + ); + } + + #[test] + fn test_lint_field_hint_explicit_fields_array() { + let mut hints = IndexMap::new(); + hints.insert( + "prompt".into(), + FieldHint { + field_type: Some(FieldType::Text), + accept: None, + placeholder: Some("Enter text".into()), + }, + ); + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + ..Default::default() + }; + let params = serde_json::json!({ + "fields": [ + { "name": "media" }, + { "name": "prompt" } + ] + }); + let nodes = vec![node("streamkit::http_input", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "field-hint-unknown-field"), + "Should not warn when hint matches declared field: {warnings:?}" + ); + } + + // Rule 16 — publish-no-transport + #[test] + fn test_lint_publish_no_transport() { + let c = dynamic_client(); + let nodes: Vec> = vec![]; // no MoQ nodes + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!(warnings.iter().any(|w| w.rule == "publish-no-transport")); + } + + #[test] + fn test_lint_publish_with_peer_clean() { + let c = dynamic_client(); + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "input", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "publish-no-transport"), + "Should not warn when peer exists: {warnings:?}" + ); + } + + // Rule 17 — watch-no-transport + #[test] + fn test_lint_watch_no_transport() { + let c = ClientSection { + gateway_path: Some("/moq/test".into()), + watch: Some(WatchConfig { broadcast: "output".into(), audio: true, video: true }), + ..Default::default() + }; + let nodes: Vec> = vec![]; // no MoQ nodes + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!(warnings.iter().any(|w| w.rule == "watch-no-transport")); + } + + // Rule 18 — gateway-path-mismatch + #[test] + fn test_lint_gateway_path_mismatch() { + let c = ClientSection { + gateway_path: Some("/moq/wrong".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + ..Default::default() + }; + let params = serde_json::json!({ + "gateway_path": "/moq/correct", + "input_broadcast": "input" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "gateway-path-mismatch"), + "Should warn when gateway_path differs: {warnings:?}" + ); + } + + #[test] + fn test_lint_gateway_path_match_clean() { + let c = dynamic_client(); // gateway_path = /moq/test + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "input", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "gateway-path-mismatch"), + "Should not warn when gateway_path matches: {warnings:?}" + ); + } + + // Rule 19 — relay-url-mismatch + #[test] + fn test_lint_relay_url_mismatch() { + let c = ClientSection { + relay_url: Some("https://relay.example.com".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + ..Default::default() + }; + let params = serde_json::json!({ + "url": "https://other-relay.example.com", + "broadcast": "input" + }); + let nodes = vec![node("transport::moq::publisher", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "relay-url-mismatch"), + "Should warn when relay_url differs: {warnings:?}" + ); + } + + #[test] + fn test_lint_relay_url_match_clean() { + let c = ClientSection { + relay_url: Some("https://relay.example.com".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + ..Default::default() + }; + let params = serde_json::json!({ + "url": "https://relay.example.com", + "broadcast": "input" + }); + let nodes = vec![node("transport::moq::subscriber", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "relay-url-mismatch"), + "Should not warn when relay_url matches: {warnings:?}" + ); + } + + // Rule 20 — broadcast-mismatch + #[test] + fn test_lint_broadcast_mismatch_publish() { + let c = ClientSection { + gateway_path: Some("/moq/test".into()), + publish: Some(PublishConfig { + broadcast: "wrong_name".into(), + audio: true, + video: false, + }), + ..Default::default() + }; + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "camera", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "broadcast-mismatch"), + "Should warn when publish.broadcast doesn't match any node broadcast: {warnings:?}" + ); + } + + #[test] + fn test_lint_broadcast_mismatch_watch() { + let c = ClientSection { + gateway_path: Some("/moq/test".into()), + watch: Some(WatchConfig { broadcast: "wrong_name".into(), audio: true, video: true }), + ..Default::default() + }; + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "camera", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "broadcast-mismatch"), + "Should warn when watch.broadcast doesn't match any node broadcast: {warnings:?}" + ); + } + + #[test] + fn test_lint_broadcast_match_clean() { + let c = dynamic_client(); // publish=input, watch=output + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "input", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "broadcast-mismatch"), + "Should not warn when broadcast names match: {warnings:?}" + ); + } } diff --git a/crates/engine/benches/compositor_pipeline.rs b/crates/engine/benches/compositor_pipeline.rs index bdd3789c..5c2347db 100644 --- a/crates/engine/benches/compositor_pipeline.rs +++ b/crates/engine/benches/compositor_pipeline.rs @@ -254,6 +254,7 @@ fn build_pipeline(width: u32, height: u32, fps: u32, frame_count: u32) -> stream nodes, connections, view_data: None, + client: None, } } diff --git a/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml b/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml index fe44c9f9..60c1860a 100644 --- a/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml +++ b/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml @@ -4,7 +4,6 @@ # Purpose: maximize throughput/CPU load by removing real-time pacer nodes. # This intentionally processes inputs as fast as possible. # -# skit:input_asset_tags=speech # name: "Loadtest: Audio Mixing (No Pacer)" diff --git a/samples/pipelines/dynamic/moq.yml b/samples/pipelines/dynamic/moq.yml index 4a7f139f..484f1ac7 100644 --- a/samples/pipelines/dynamic/moq.yml +++ b/samples/pipelines/dynamic/moq.yml @@ -1,6 +1,16 @@ name: Real-Time MoQ Transcoder description: Decodes a MoQ input broadcast, applies gain, and republishes to MoQ mode: dynamic +client: + gateway_path: /moq/transcoder + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: moq_peer: kind: transport::moq::peer diff --git a/samples/pipelines/dynamic/moq_mixing.yml b/samples/pipelines/dynamic/moq_mixing.yml index 88fca62f..c14ee79f 100644 --- a/samples/pipelines/dynamic/moq_mixing.yml +++ b/samples/pipelines/dynamic/moq_mixing.yml @@ -1,6 +1,16 @@ name: Real-Time MoQ Mixing (Mic + Music) description: Mixes a live MoQ input stream with a local music track and republishes to MoQ mode: dynamic +client: + gateway_path: /moq/mixing + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ # INPUTS: MoQ mic input + local music file diff --git a/samples/pipelines/dynamic/moq_peer.yml b/samples/pipelines/dynamic/moq_peer.yml index caaafac4..8213d58f 100644 --- a/samples/pipelines/dynamic/moq_peer.yml +++ b/samples/pipelines/dynamic/moq_peer.yml @@ -5,6 +5,16 @@ name: MoQ Peer Transcoder (Gateway) description: Accepts a WebTransport client via transport::moq::peer, applies gain, and loops processed audio back to subscribers mode: dynamic +client: + gateway_path: /moq + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # Bidirectional node that accepts WebTransport connections and provides: diff --git a/samples/pipelines/dynamic/moq_relay_echo.yml b/samples/pipelines/dynamic/moq_relay_echo.yml index ac398037..21311187 100644 --- a/samples/pipelines/dynamic/moq_relay_echo.yml +++ b/samples/pipelines/dynamic/moq_relay_echo.yml @@ -5,6 +5,16 @@ name: MoQ Relay Echo (Audio) description: Subscribes to an audio broadcast on a MoQ relay, applies unity gain, and republishes — validates the full relay pub/sub path mode: dynamic +client: + relay_url: http://localhost:4443 + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: moq_input: kind: transport::moq::subscriber diff --git a/samples/pipelines/dynamic/moq_relay_webcam_pip.yml b/samples/pipelines/dynamic/moq_relay_webcam_pip.yml index 8a62084a..54038e4f 100644 --- a/samples/pipelines/dynamic/moq_relay_webcam_pip.yml +++ b/samples/pipelines/dynamic/moq_relay_webcam_pip.yml @@ -7,6 +7,16 @@ description: Subscribes audio+video from a MoQ relay, composites the webcam as P # NOTE: The publisher must already be streaming to the 'input' broadcast before # starting this pipeline, otherwise track discovery will miss the video/data pin. mode: dynamic +client: + relay_url: http://localhost:4443 + publish: + broadcast: input + audio: true + video: true + watch: + broadcast: output + audio: true + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/speech-translate-en-es.yaml b/samples/pipelines/dynamic/speech-translate-en-es.yaml index b5bcf74b..603f6f16 100644 --- a/samples/pipelines/dynamic/speech-translate-en-es.yaml +++ b/samples/pipelines/dynamic/speech-translate-en-es.yaml @@ -23,6 +23,16 @@ name: Real-Time Speech Translation (English → Spanish) description: Real-time speech translation from English to Spanish via MoQ mode: dynamic +client: + gateway_path: /moq/speech-translate-en-es + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/speech-translate-es-en.yaml b/samples/pipelines/dynamic/speech-translate-es-en.yaml index 4e5cd217..27b7228f 100644 --- a/samples/pipelines/dynamic/speech-translate-es-en.yaml +++ b/samples/pipelines/dynamic/speech-translate-es-en.yaml @@ -24,6 +24,16 @@ name: Real-Time Speech Translation (Spanish → English) description: Real-time speech translation from Spanish to English via MoQ mode: dynamic +client: + gateway_path: /moq/speech-translate-es-en + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml b/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml index 111a73bb..b1056757 100644 --- a/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml +++ b/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml @@ -27,6 +27,16 @@ name: Real-Time Speech Translation (English → Spanish) - Helsinki description: Real-time speech translation from English to Spanish using Apache 2.0 licensed Helsinki OPUS-MT mode: dynamic +client: + gateway_path: /moq/speech-translate-helsinki-en-es + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml b/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml index 61d959ed..bed2ea7a 100644 --- a/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml +++ b/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml @@ -27,6 +27,16 @@ name: Real-Time Speech Translation (Spanish → English) - Helsinki description: Real-time speech translation from Spanish to English using Apache 2.0 licensed Helsinki OPUS-MT mode: dynamic +client: + gateway_path: /moq/speech-translate-helsinki-es-en + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/video_moq_colorbars.yml b/samples/pipelines/dynamic/video_moq_colorbars.yml index 1a4921ef..eb18cf84 100644 --- a/samples/pipelines/dynamic/video_moq_colorbars.yml +++ b/samples/pipelines/dynamic/video_moq_colorbars.yml @@ -5,6 +5,12 @@ name: Video Color Bars (MoQ Stream) description: Continuously generates SMPTE color bars and streams via MoQ mode: dynamic +client: + gateway_path: /moq/video + watch: + broadcast: output + audio: false + video: true nodes: colorbars: diff --git a/samples/pipelines/dynamic/video_moq_compositor.yml b/samples/pipelines/dynamic/video_moq_compositor.yml index 22ca5c1b..aa5d96c6 100644 --- a/samples/pipelines/dynamic/video_moq_compositor.yml +++ b/samples/pipelines/dynamic/video_moq_compositor.yml @@ -5,6 +5,12 @@ name: Video Compositor (MoQ Stream) description: Composites two colorbars sources through the compositor node and streams via MoQ mode: dynamic +client: + gateway_path: /moq/video + watch: + broadcast: output + audio: false + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml b/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml index 4c4d6c6d..37fc9f09 100644 --- a/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml +++ b/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml @@ -5,6 +5,16 @@ name: Webcam Circle PiP (MoQ Stream) description: Composites the user's webcam as a circular picture-in-picture overlay (Loom-style) over colorbars, loops audio through a gain filter, and streams both via MoQ mode: dynamic +client: + gateway_path: /moq/video + publish: + broadcast: input + audio: true + video: true + watch: + broadcast: output + audio: true + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/video_moq_webcam_pip.yml b/samples/pipelines/dynamic/video_moq_webcam_pip.yml index b342adbf..184d8642 100644 --- a/samples/pipelines/dynamic/video_moq_webcam_pip.yml +++ b/samples/pipelines/dynamic/video_moq_webcam_pip.yml @@ -5,6 +5,16 @@ name: Webcam PiP (MoQ Stream) description: Composites the user's webcam as picture-in-picture over colorbars with a text overlay, loops audio through a gain filter, and streams both via MoQ mode: dynamic +client: + gateway_path: /moq/video + publish: + broadcast: input + audio: true + video: true + watch: + broadcast: output + audio: true + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/voice-agent-openai.yaml b/samples/pipelines/dynamic/voice-agent-openai.yaml index 4a73d601..63ba61c5 100644 --- a/samples/pipelines/dynamic/voice-agent-openai.yaml +++ b/samples/pipelines/dynamic/voice-agent-openai.yaml @@ -12,6 +12,16 @@ name: Real-Time Voice Agent (OpenAI) description: Runs a MoQ voice agent using Whisper STT, OpenAI, and Kokoro TTS mode: dynamic +client: + gateway_path: /moq/voice-agent-openai + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ # INPUT: Receive audio from MoQ broadcast diff --git a/samples/pipelines/dynamic/voice-weather-open-meteo.yaml b/samples/pipelines/dynamic/voice-weather-open-meteo.yaml index f5590e89..d4e6ce13 100644 --- a/samples/pipelines/dynamic/voice-weather-open-meteo.yaml +++ b/samples/pipelines/dynamic/voice-weather-open-meteo.yaml @@ -30,6 +30,16 @@ name: Real-Time Voice Weather (Open-Meteo) description: Ask for the weather in a location (STT → Open-Meteo → TTS) via MoQ mode: dynamic +client: + gateway_path: /moq/voice-weather-open-meteo + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ # INPUT: Receive audio from MoQ broadcast diff --git a/samples/pipelines/oneshot/double_volume.yml b/samples/pipelines/oneshot/double_volume.yml index a5890c07..ac7b48dc 100644 --- a/samples/pipelines/oneshot/double_volume.yml +++ b/samples/pipelines/oneshot/double_volume.yml @@ -1,6 +1,12 @@ name: Volume Boost (2×) description: Boosts the volume of an uploaded Ogg/Opus file and returns Ogg/Opus mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + output: + type: audio steps: - kind: streamkit::http_input - kind: containers::ogg::demuxer diff --git a/samples/pipelines/oneshot/dual_upload_mixing.yml b/samples/pipelines/oneshot/dual_upload_mixing.yml index d02534be..d15c5dd6 100644 --- a/samples/pipelines/oneshot/dual_upload_mixing.yml +++ b/samples/pipelines/oneshot/dual_upload_mixing.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Dual Upload Mixer description: Mix two uploaded Ogg/Opus tracks and return Opus/WebM mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: audio nodes: # ============================================================ # INPUTS: Two uploaded audio tracks diff --git a/samples/pipelines/oneshot/gain_filter_rust.yml b/samples/pipelines/oneshot/gain_filter_rust.yml index 70293187..44bd3f77 100644 --- a/samples/pipelines/oneshot/gain_filter_rust.yml +++ b/samples/pipelines/oneshot/gain_filter_rust.yml @@ -1,6 +1,12 @@ name: Gain Filter (Rust WASM) description: Applies a gain filter using the Rust WASM plugin mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + output: + type: audio steps: - kind: streamkit::http_input - kind: containers::ogg::demuxer diff --git a/samples/pipelines/oneshot/kokoro-tts.yml b/samples/pipelines/oneshot/kokoro-tts.yml index 0c88367b..f438c802 100644 --- a/samples/pipelines/oneshot/kokoro-tts.yml +++ b/samples/pipelines/oneshot/kokoro-tts.yml @@ -1,6 +1,12 @@ name: Text-to-Speech (Kokoro) description: Synthesizes speech from text using Kokoro mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/matcha-tts.yml b/samples/pipelines/oneshot/matcha-tts.yml index 517743df..f8a01a8a 100644 --- a/samples/pipelines/oneshot/matcha-tts.yml +++ b/samples/pipelines/oneshot/matcha-tts.yml @@ -5,6 +5,12 @@ name: Text-to-Speech (Matcha) description: Synthesizes speech from text using Matcha mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/mixing.yml b/samples/pipelines/oneshot/mixing.yml index 2b98df29..069edcaf 100644 --- a/samples/pipelines/oneshot/mixing.yml +++ b/samples/pipelines/oneshot/mixing.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Audio Mixing (Upload + Music Track) description: Mixes uploaded audio with a built-in music track and returns Opus/WebM mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: audio nodes: # ============================================================ # INPUTS: HTTP upload + local music file diff --git a/samples/pipelines/oneshot/piper-tts.yml b/samples/pipelines/oneshot/piper-tts.yml index 4f9e0c48..40692f4f 100644 --- a/samples/pipelines/oneshot/piper-tts.yml +++ b/samples/pipelines/oneshot/piper-tts.yml @@ -5,6 +5,12 @@ name: Text-to-Speech (Piper) description: Synthesizes speech from text using Piper mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/pocket-tts-voice-clone.yml b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml index 095714e9..5bbbcf38 100644 --- a/samples/pipelines/oneshot/pocket-tts-voice-clone.yml +++ b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml @@ -2,11 +2,21 @@ # # SPDX-License-Identifier: MPL-2.0 -# skit:input_formats=voice:wav - name: Text-to-Speech (Pocket TTS Voice Clone) description: Synthesizes speech from text using Pocket TTS with a voice prompt WAV mode: oneshot +client: + input: + type: text + field_hints: + voice: + type: file + accept: "audio/wav" + text: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio nodes: uploads: kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/pocket-tts.yml b/samples/pipelines/oneshot/pocket-tts.yml index ae088dd7..0981de9f 100644 --- a/samples/pipelines/oneshot/pocket-tts.yml +++ b/samples/pipelines/oneshot/pocket-tts.yml @@ -5,6 +5,12 @@ name: Text-to-Speech (Pocket TTS) description: Synthesizes speech from text using Pocket TTS mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/sensevoice-stt.yml b/samples/pipelines/oneshot/sensevoice-stt.yml index 25b197e8..57cd82a4 100644 --- a/samples/pipelines/oneshot/sensevoice-stt.yml +++ b/samples/pipelines/oneshot/sensevoice-stt.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Speech-to-Text (SenseVoice) description: Transcribes speech in multiple languages using SenseVoice mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: transcription steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/speech_to_text.yml b/samples/pipelines/oneshot/speech_to_text.yml index f772a676..8d797f5f 100644 --- a/samples/pipelines/oneshot/speech_to_text.yml +++ b/samples/pipelines/oneshot/speech_to_text.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Speech-to-Text (Whisper) description: Transcribes speech to text using Whisper mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: transcription steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/speech_to_text_translate.yml b/samples/pipelines/oneshot/speech_to_text_translate.yml index e0eede96..efbe1697 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate.yml @@ -2,12 +2,17 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - name: Speech Translation (English → Spanish) description: Translates English speech into Spanish speech mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: audio steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml index 11c96746..5f489f65 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml @@ -2,9 +2,6 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - # Speech Translation using Helsinki-NLP OPUS-MT (Apache 2.0 licensed) # # This pipeline is an alternative to speech_to_text_translate.yml that uses @@ -18,6 +15,14 @@ name: Speech Translation (English → Spanish) - Helsinki description: Translates English speech into Spanish speech using Apache 2.0 licensed Helsinki OPUS-MT mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: audio steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/supertonic-tts.yml b/samples/pipelines/oneshot/supertonic-tts.yml index ca793cd9..1dcc8019 100644 --- a/samples/pipelines/oneshot/supertonic-tts.yml +++ b/samples/pipelines/oneshot/supertonic-tts.yml @@ -1,6 +1,12 @@ name: Text-to-Speech (Supertonic) description: Synthesizes speech from text using Supertonic (5 languages, 10 voice styles) mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/useless-facts-tts.yml b/samples/pipelines/oneshot/useless-facts-tts.yml index ee252c5b..91525dbc 100644 --- a/samples/pipelines/oneshot/useless-facts-tts.yml +++ b/samples/pipelines/oneshot/useless-facts-tts.yml @@ -5,6 +5,11 @@ name: Text-to-Speech (Useless Facts) description: Fetches a random fact and reads it out loud using Kokoro mode: oneshot +client: + input: + type: trigger + output: + type: audio steps: # HTTP input (body can be empty, just triggers the pipeline) - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/vad-demo.yml b/samples/pipelines/oneshot/vad-demo.yml index 1d9b4bde..dad8056b 100644 --- a/samples/pipelines/oneshot/vad-demo.yml +++ b/samples/pipelines/oneshot/vad-demo.yml @@ -2,12 +2,17 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - name: Voice Activity Detection description: Detects voice activity and outputs events as JSON mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: json steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/vad-filtered-stt.yml b/samples/pipelines/oneshot/vad-filtered-stt.yml index e3395b5f..e423285e 100644 --- a/samples/pipelines/oneshot/vad-filtered-stt.yml +++ b/samples/pipelines/oneshot/vad-filtered-stt.yml @@ -2,12 +2,17 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - name: Speech-to-Text (Whisper, VAD-Filtered) description: Filters silence with VAD before transcribing with Whisper mode: oneshot +client: + input: + type: file_upload + accept: "audio/opus" + asset_tags: + - speech + output: + type: transcription steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/video_colorbars.yml b/samples/pipelines/oneshot/video_colorbars.yml index de72e242..24bd1d5e 100644 --- a/samples/pipelines/oneshot/video_colorbars.yml +++ b/samples/pipelines/oneshot/video_colorbars.yml @@ -5,6 +5,11 @@ name: Video Color Bars (VP9/WebM) description: Generates SMPTE color bars, encodes to VP9, and outputs a WebM file (5 seconds) mode: oneshot +client: + input: + type: none + output: + type: video nodes: colorbars: diff --git a/samples/pipelines/oneshot/video_compositor_demo.yml b/samples/pipelines/oneshot/video_compositor_demo.yml index e41beefd..95e1b277 100644 --- a/samples/pipelines/oneshot/video_compositor_demo.yml +++ b/samples/pipelines/oneshot/video_compositor_demo.yml @@ -16,6 +16,11 @@ name: Video Compositor Demo description: Composites two colorbars sources (main + PiP) with a text overlay through the compositor node, encodes to VP9, and streams a WebM via http_output with real-time pacing (3 seconds) mode: oneshot +client: + input: + type: none + output: + type: video nodes: colorbars_bg: diff --git a/ui/src/hooks/useConvertViewState.ts b/ui/src/hooks/useConvertViewState.ts index 839ba034..7f201b44 100644 --- a/ui/src/hooks/useConvertViewState.ts +++ b/ui/src/hooks/useConvertViewState.ts @@ -30,9 +30,6 @@ export function useConvertViewState() { // Pipeline state const [pipelineYaml, setPipelineYaml] = useState(''); const [selectedTemplateId, setSelectedTemplateId] = useState(''); - const [isTranscriptionPipeline, setIsTranscriptionPipeline] = useState(false); - const [isTTSPipeline, setIsTTSPipeline] = useState(false); - const [isNoInputPipeline, setIsNoInputPipeline] = useState(false); const [textInput, setTextInput] = useState(''); // Conversion state @@ -73,12 +70,6 @@ export function useConvertViewState() { setPipelineYaml, selectedTemplateId, setSelectedTemplateId, - isTranscriptionPipeline, - setIsTranscriptionPipeline, - isTTSPipeline, - setIsTTSPipeline, - isNoInputPipeline, - setIsNoInputPipeline, textInput, setTextInput, diff --git a/ui/src/hooks/useMonitorPreview.ts b/ui/src/hooks/useMonitorPreview.ts index 0a6dc39b..bfc9e012 100644 --- a/ui/src/hooks/useMonitorPreview.ts +++ b/ui/src/hooks/useMonitorPreview.ts @@ -18,6 +18,45 @@ import { useStreamStore } from '@/stores/streamStore'; import type { Pipeline } from '@/types/types'; import { updateUrlPath } from '@/utils/moqPeerSettings'; +interface PreviewMoqConfig { + gatewayPath?: string; + outputBroadcast?: string; + outputsAudio: boolean; + outputsVideo: boolean; +} + +/** + * Derives MoQ preview configuration from the pipeline's node graph. + * Used as a fallback for interactively-created sessions that don't have + * a `client` section. + */ +function deriveMoqConfigFromNodes(pipeline: Pipeline): PreviewMoqConfig { + const config: PreviewMoqConfig = { outputsAudio: true, outputsVideo: true }; + + const moqEntry = Object.entries(pipeline.nodes).find( + ([, n]) => n.kind === 'transport::moq::peer' && n.params + ); + if (!moqEntry) return config; + + const [moqNodeName, moqNode] = moqEntry; + const params = moqNode.params as Record; + config.gatewayPath = params.gateway_path as string | undefined; + config.outputBroadcast = params.output_broadcast as string | undefined; + + // Detect media types from connection graph + config.outputsAudio = false; + config.outputsVideo = false; + for (const conn of pipeline.connections) { + if (conn.to_node !== moqNodeName) continue; + const sourceNode = pipeline.nodes[conn.from_node]; + if (!sourceNode?.kind) continue; + if (sourceNode.kind.startsWith('audio::')) config.outputsAudio = true; + else if (sourceNode.kind.startsWith('video::')) config.outputsVideo = true; + } + + return config; +} + export interface UseMonitorPreviewReturn { isPreviewConnected: boolean; handleStartPreview: () => Promise; @@ -69,8 +108,8 @@ export function useMonitorPreview( } }, [selectedSessionId, previewStatus, previewDisconnect]); - // Extract MoQ peer settings from the selected session's pipeline so the - // preview connects to the correct gateway path and output broadcast. + // Read MoQ peer settings from the pipeline's declarative client section + // instead of scanning the compiled node graph. const handleStartPreview = useCallback(async () => { // Configure for watch-only mode (no publish/mic) previewSetEnablePublish(false); @@ -79,46 +118,40 @@ export function useMonitorPreview( await previewLoadConfig(); } - // Extract gateway_path and output_broadcast from the pipeline's moq_peer node - let moqNodeName: string | undefined; - const moqNode = pipeline - ? Object.entries(pipeline.nodes).find( - ([, n]) => n.kind === 'transport::moq::peer' && n.params - ) - : undefined; - if (moqNode) { - moqNodeName = moqNode[0]; - const params = moqNode[1].params as Record; - const gatewayPath = params.gateway_path as string | undefined; - const outputBroadcast = params.output_broadcast as string | undefined; - // Read serverUrl at call-time via getState() rather than via a - // hook selector — this is a standard Zustand pattern for values - // that should be fresh when the callback fires, not stale from - // the last render. - const currentUrl = useStreamStore.getState().serverUrl; - if (gatewayPath && currentUrl) { - previewSetServerUrl(updateUrlPath(currentUrl, gatewayPath)); - } - if (outputBroadcast) { - previewSetOutputBroadcast(outputBroadcast); - } - } - - // Detect which media types the pipeline outputs by checking the kinds of - // nodes connected to the moq_peer's input pins. + // Read gateway_path and output_broadcast from the pipeline's client section. + // Fall back to scanning the node graph for interactively-created sessions + // that don't have a client section. + const client = pipeline?.client ?? null; + let gatewayPath: string | undefined; + let outputBroadcast: string | undefined; let outputsAudio = true; let outputsVideo = true; - if (pipeline && moqNodeName) { - outputsAudio = false; - outputsVideo = false; - for (const conn of pipeline.connections) { - if (conn.to_node !== moqNodeName) continue; - const sourceNode = pipeline.nodes[conn.from_node]; - if (!sourceNode?.kind) continue; - if (sourceNode.kind.startsWith('audio::')) outputsAudio = true; - else if (sourceNode.kind.startsWith('video::')) outputsVideo = true; + + if (client) { + gatewayPath = client.gateway_path ?? undefined; + outputBroadcast = client.watch?.broadcast; + outputsAudio = client.watch?.audio ?? false; + outputsVideo = client.watch?.video ?? false; + } else if (pipeline) { + const fallback = deriveMoqConfigFromNodes(pipeline); + gatewayPath = fallback.gatewayPath; + outputBroadcast = fallback.outputBroadcast; + outputsAudio = fallback.outputsAudio; + outputsVideo = fallback.outputsVideo; + } + + if (gatewayPath) { + // Use the original config URL as the base so that the preview URL + // isn't polluted by a relay URL the user previously selected. + const baseUrl = + useStreamStore.getState().configServerUrl || useStreamStore.getState().serverUrl; + if (baseUrl) { + previewSetServerUrl(updateUrlPath(baseUrl, gatewayPath)); } } + if (outputBroadcast) { + previewSetOutputBroadcast(outputBroadcast); + } previewSetPipelineOutputTypes(outputsAudio, outputsVideo); await previewConnect(); diff --git a/ui/src/services/websocket.test.ts b/ui/src/services/websocket.test.ts index 094f2c5f..3e8b7258 100644 --- a/ui/src/services/websocket.test.ts +++ b/ui/src/services/websocket.test.ts @@ -325,6 +325,7 @@ describe('WebSocketService', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }, diff --git a/ui/src/stores/sessionStore.edge-cases.test.ts b/ui/src/stores/sessionStore.edge-cases.test.ts index 03593945..cd89ff21 100644 --- a/ui/src/stores/sessionStore.edge-cases.test.ts +++ b/ui/src/stores/sessionStore.edge-cases.test.ts @@ -50,6 +50,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'core::passthrough', params: {}, state: 'Initializing' } }, connections: [], }; @@ -57,6 +58,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-2': { kind: 'core::gain', params: { gain: 1.0 }, state: 'Initializing' } }, connections: [], }; @@ -98,6 +100,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'core::passthrough', @@ -236,6 +239,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { compositor: { kind: 'video::compositor', @@ -266,6 +270,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], view_data: { nodeA: { key: 'original' } }, @@ -276,6 +281,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], view_data: { nodeB: { key: 'new' } }, @@ -292,6 +298,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -310,6 +317,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic' as const, + client: null, nodes: {}, connections: [], view_data: { comp: { layers: { in_0: { x: 10 } } } }, @@ -330,6 +338,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'core::passthrough', params: {}, state: 'Initializing' }, }, diff --git a/ui/src/stores/sessionStore.test.ts b/ui/src/stores/sessionStore.test.ts index ce5f86d9..8b5ea854 100644 --- a/ui/src/stores/sessionStore.test.ts +++ b/ui/src/stores/sessionStore.test.ts @@ -107,6 +107,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -129,6 +130,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -142,6 +144,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -179,6 +182,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -200,6 +204,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -223,6 +228,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -254,6 +260,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -279,6 +286,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [ { @@ -338,6 +346,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -362,6 +371,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; diff --git a/ui/src/stores/streamStore.test.ts b/ui/src/stores/streamStore.test.ts index 4a40674c..3690cd50 100644 --- a/ui/src/stores/streamStore.test.ts +++ b/ui/src/stores/streamStore.test.ts @@ -98,6 +98,9 @@ describe('streamStore', () => { pipelineOutputsVideo: true, errorMessage: '', configLoaded: false, + configServerUrl: '', + connectAbort: null, + connectingStep: '', activeSessionId: null, activeSessionName: null, activePipelineName: null, @@ -149,6 +152,12 @@ describe('streamStore', () => { expect(state.activePipelineName).toBeNull(); }); + it('should have null connectAbort and empty connectingStep initially', () => { + const state = useStreamStore.getState(); + expect(state.connectAbort).toBeNull(); + expect(state.connectingStep).toBe(''); + }); + it('should have null MoQ references initially', () => { const state = useStreamStore.getState(); expect(state.publish).toBeNull(); @@ -276,9 +285,29 @@ describe('streamStore', () => { const state = useStreamStore.getState(); expect(state.serverUrl).toBe('http://config-server.com:9000/moq'); + expect(state.configServerUrl).toBe('http://config-server.com:9000/moq'); expect(state.configLoaded).toBe(true); }); + it('should preserve configServerUrl when serverUrl is overwritten by relay', async () => { + const mockConfig = { + moqGatewayUrl: 'http://gateway.example.com:4545/moq', + }; + + vi.mocked(configService.fetchConfig).mockResolvedValue(mockConfig); + + const { loadConfig } = useStreamStore.getState(); + await loadConfig(); + + // Simulate selecting a relay pipeline which overwrites serverUrl + useStreamStore.getState().setServerUrl('http://relay.example.com:4443'); + + const state = useStreamStore.getState(); + expect(state.serverUrl).toBe('http://relay.example.com:4443'); + // configServerUrl must remain the original gateway URL + expect(state.configServerUrl).toBe('http://gateway.example.com:4545/moq'); + }); + it('should set configLoaded when no moqGatewayUrl in config', async () => { const mockConfig = {}; @@ -497,6 +526,34 @@ describe('streamStore', () => { expect(useStreamStore.getState().status).toBe('disconnected'); }); + + it('should abort in-flight connect attempt on disconnect', () => { + const abort = new AbortController(); + const abortSpy = vi.spyOn(abort, 'abort'); + useStreamStore.setState({ + status: 'connecting', + connectAbort: abort, + }); + + useStreamStore.getState().disconnect(); + + expect(abortSpy).toHaveBeenCalled(); + const state = useStreamStore.getState(); + expect(state.status).toBe('disconnected'); + expect(state.connectAbort).toBeNull(); + expect(state.connectingStep).toBe(''); + }); + + it('should clear connectingStep on disconnect', () => { + useStreamStore.setState({ + status: 'connecting', + connectingStep: 'devices', + }); + + useStreamStore.getState().disconnect(); + + expect(useStreamStore.getState().connectingStep).toBe(''); + }); }); describe('connection state machine', () => { diff --git a/ui/src/stores/streamStore.ts b/ui/src/stores/streamStore.ts index 082268d7..b4367011 100644 --- a/ui/src/stores/streamStore.ts +++ b/ui/src/stores/streamStore.ts @@ -63,12 +63,19 @@ interface StreamState { // Config state configLoaded: boolean; + /** The original gateway URL from /api/v1/config, preserved across pipeline switches. */ + configServerUrl: string; // Active session state (persisted) activeSessionId: string | null; activeSessionName: string | null; activePipelineName: string | null; + // Connect lifecycle — abort controller for the in-flight performConnect call + connectAbort: AbortController | null; + /** Human-readable label for the current phase of a connect attempt. */ + connectingStep: string; + // MoQ references (stored but not serialized) publish: Publish.Broadcast | null; watch: Watch.Broadcast | null; @@ -148,6 +155,9 @@ export const useStreamStore = create((set, get) => ({ isExternalRelay: false, errorMessage: '', configLoaded: false, + configServerUrl: '', + connectAbort: null, + connectingStep: '', // Active session state activeSessionId: null, @@ -188,7 +198,11 @@ export const useStreamStore = create((set, get) => ({ try { const config = await fetchConfig(); if (config.moqGatewayUrl) { - set({ serverUrl: config.moqGatewayUrl, configLoaded: true }); + set({ + serverUrl: config.moqGatewayUrl, + configServerUrl: config.moqGatewayUrl, + configLoaded: true, + }); } else { set({ configLoaded: true, @@ -235,7 +249,15 @@ export const useStreamStore = create((set, get) => ({ return false; } + // Abort any in-flight connect attempt so it doesn't later overwrite + // our state. This is defensive — normally disconnect() already aborted, + // but a rapid connect→connect without disconnect can still race. + state.connectAbort?.abort(); + + const abort = new AbortController(); set({ + connectAbort: abort, + connectingStep: '', status: 'connecting', errorMessage: '', watchStatus: decision.shouldWatch ? 'loading' : 'disabled', @@ -243,17 +265,23 @@ export const useStreamStore = create((set, get) => ({ cameraStatus: decision.shouldPublish && state.pipelineNeedsVideo ? 'requesting' : 'disabled', }); - return performConnect(state, decision, get, set); + return performConnect(state, decision, get, set, abort.signal); }, disconnect: () => { const state = get(); + // Cancel any in-flight connect attempt so its catch/success path + // doesn't later overwrite the freshly-reset store. + state.connectAbort?.abort(); + // Reuse the same teardown logic used when a connect attempt fails. cleanupConnectAttempt(state); set({ status: 'disconnected', + connectAbort: null, + connectingStep: '', isMicEnabled: false, micStatus: 'disabled', isCameraEnabled: false, diff --git a/ui/src/stores/streamStoreHelpers.test.ts b/ui/src/stores/streamStoreHelpers.test.ts index d3ca93a2..45f0e73b 100644 --- a/ui/src/stores/streamStoreHelpers.test.ts +++ b/ui/src/stores/streamStoreHelpers.test.ts @@ -299,6 +299,53 @@ describe('waitForSignalValue', () => { vi.useRealTimers(); }); + + it('should reject immediately when abortSignal is already aborted', async () => { + const signal = createMockSignal(0); + const abort = new AbortController(); + abort.abort(); + + await expect( + waitForSignalValue(signal, (v) => v > 0, 5_000, 'timeout', abort.signal) + ).rejects.toThrow('Aborted'); + }); + + it('should reject with AbortError when abortSignal fires during wait', async () => { + const signal = createMockSignal(0); + const abort = new AbortController(); + + const promise = waitForSignalValue(signal, (v) => v > 0, 5_000, 'timeout', abort.signal); + + // Abort before the signal value changes + abort.abort(); + + await expect(promise).rejects.toThrow('Aborted'); + }); + + it('should not reject on abort if predicate already matched', async () => { + const signal = createMockSignal(42); + const abort = new AbortController(); + + // Predicate matches initial value — resolves synchronously before abort + const value = await waitForSignalValue(signal, (v) => v === 42, 5_000, 'timeout', abort.signal); + expect(value).toBe(42); + + // Aborting after resolution should be harmless + abort.abort(); + }); + + it('should clean up subscription when abortSignal fires', async () => { + const signal = createMockSignal(0); + const abort = new AbortController(); + + const promise = waitForSignalValue(signal, (v) => v > 100, 5_000, 'timeout', abort.signal); + abort.abort(); + + await expect(promise).rejects.toThrow('Aborted'); + + // After abort, emitting new values should be harmless (no dangling listeners) + expect(() => signal.set(200)).not.toThrow(); + }); }); // --------------------------------------------------------------------------- diff --git a/ui/src/stores/streamStoreHelpers.ts b/ui/src/stores/streamStoreHelpers.ts index b80e7e6f..97244030 100644 --- a/ui/src/stores/streamStoreHelpers.ts +++ b/ui/src/stores/streamStoreHelpers.ts @@ -64,6 +64,9 @@ export interface ConnectableState { micStatus: MicStatus; cameraStatus: CameraStatus; watchStatus: WatchStatus; + /** Human-readable label for the current phase of a connect attempt + * (e.g. 'devices', 'relay', 'pipeline'). Empty when idle. */ + connectingStep: string; } type StateSetter = (partial: Partial) => void; @@ -89,8 +92,13 @@ export function waitForSignalValue( signal: Getter, predicate: (value: T) => boolean, timeoutMs: number, - timeoutMessage: string + timeoutMessage: string, + abortSignal?: AbortSignal ): Promise { + if (abortSignal?.aborted) { + return Promise.reject(new DOMException('Aborted', 'AbortError')); + } + const initial = signal.peek(); if (predicate(initial)) { return Promise.resolve(initial); @@ -98,15 +106,31 @@ export function waitForSignalValue( return new Promise((resolve, reject) => { let dispose: () => void = () => {}; - const timeoutId = setTimeout(() => { + + const cleanup = () => { + clearTimeout(timeoutId); dispose(); + }; + + const timeoutId = setTimeout(() => { + cleanup(); reject(new Error(timeoutMessage)); }, timeoutMs); + if (abortSignal) { + abortSignal.addEventListener( + 'abort', + () => { + cleanup(); + reject(new DOMException('Aborted', 'AbortError')); + }, + { once: true } + ); + } + dispose = signal.subscribe((value) => { if (predicate(value)) { - clearTimeout(timeoutId); - dispose(); + cleanup(); resolve(value); } }); @@ -122,20 +146,39 @@ export function waitForSignalValue( async function waitForBroadcastAnnouncement( connection: Hang.Moq.Connection.Reload, broadcastName: string, - timeoutMs = 15_000 + timeoutMs = 15_000, + abortSignal?: AbortSignal ): Promise { + if (abortSignal?.aborted) throw new DOMException('Aborted', 'AbortError'); const conn = connection.established.peek(); if (!conn) return; logger.info(`Waiting for broadcast '${broadcastName}' announcement...`); const announcements = conn.announced(); const deadline = Date.now() + timeoutMs; try { + // Build a promise that rejects when the abort signal fires so the + // Promise.race below reacts immediately instead of polling. + const abortPromise = abortSignal + ? new Promise((_, reject) => { + abortSignal.addEventListener( + 'abort', + () => reject(new DOMException('Aborted', 'AbortError')), + { once: true } + ); + }) + : null; + while (Date.now() < deadline) { const remaining = deadline - Date.now(); - const entry = await Promise.race([ + const racers: Promise[] = [ announcements.next(), new Promise((r) => setTimeout(() => r(null), remaining)), - ]); + ]; + if (abortPromise) racers.push(abortPromise); + + const entry = (await Promise.race(racers)) as Awaited< + ReturnType + > | null; if (!entry) break; if (entry.active && entry.path.toString() === broadcastName) { logger.info(`Broadcast '${broadcastName}' announced`); @@ -320,7 +363,8 @@ async function setupMediaSources( healthEffect: Effect, needsAudio: boolean, needsVideo: boolean, - set: StateSetter + set: StateSetter, + abortSignal?: AbortSignal ): Promise<{ microphone: Publish.Source.Microphone | null; camera: Publish.Source.Camera | null; @@ -346,7 +390,8 @@ async function setupMediaSources( camera.source, (v) => v !== undefined, 15_000, - 'Camera not available' + 'Camera not available', + abortSignal ); } catch (e) { shutdownMediaSource(camera); @@ -364,13 +409,20 @@ async function setupPublishPath( inputBroadcast: string, needsAudio: boolean, needsVideo: boolean, - set: StateSetter + set: StateSetter, + abortSignal?: AbortSignal ): Promise<{ microphone: Publish.Source.Microphone | null; camera: Publish.Source.Camera | null; publish: Publish.Broadcast; }> { - const { microphone, camera } = await setupMediaSources(healthEffect, needsAudio, needsVideo, set); + const { microphone, camera } = await setupMediaSources( + healthEffect, + needsAudio, + needsVideo, + set, + abortSignal + ); logger.info('Step 5: Creating publish broadcast'); const broadcastConfig: ConstructorParameters[0] = { @@ -398,7 +450,8 @@ async function setupPublishPath( publish.video.catalog, (v) => v !== undefined, 10_000, - 'Video encoder failed to initialize' + 'Video encoder failed to initialize', + abortSignal ); } catch (e) { publish.close(); @@ -499,32 +552,102 @@ function applyPublishResult( attempt.publish = result.publish; } +/** Create the MoQ connection and wire up the health-status sync effect. */ +function createConnectionAndHealth( + serverUrl: string, + moqToken: string, + get: () => ConnectableState, + set: StateSetter +): { connection: Hang.Moq.Connection.Reload; healthEffect: Effect } { + logger.info('Step 1: Creating connection to relay server'); + const url = new URL(serverUrl); + const jwt = moqToken.trim(); + if (jwt) { + url.searchParams.set('jwt', jwt); + } + + const connection = new Hang.Moq.Connection.Reload({ url, enabled: true }); + const healthEffect = new Effect(); + setupConnectionStatusSync(healthEffect, connection, get, set); + return { connection, healthEffect }; +} + +/** Wait for relay connection, optionally wait for broadcast announcement, then set up watch. */ +async function connectWatchPath( + attempt: ConnectAttempt, + state: ConnectableState, + decision: Extract, + set: StateSetter, + abortSignal: AbortSignal +): Promise { + set({ connectingStep: 'relay' }); + await waitForSignalValue( + attempt.connection!.established, + (value) => value !== undefined, + 12_000, + 'Timed out connecting to MoQ gateway.', + abortSignal + ); + + if (decision.shouldWatch) { + // When publishing to an external relay, the skit pipeline needs time to + // discover input tracks, build the graph, and start publishing output. + // Wait for the output broadcast to be announced on the relay before + // subscribing, otherwise the catalog subscribe gets RESET_STREAM. + // In gateway mode the skit server manages the peer connection directly, + // so no announcement polling is needed. + if (decision.shouldPublish && state.isExternalRelay) { + set({ connectingStep: 'pipeline' }); + await waitForBroadcastAnnouncement( + attempt.connection!, + state.outputBroadcast, + 15_000, + abortSignal + ); + } + + applyWatchResult( + attempt, + setupWatchPath( + attempt.healthEffect!, + attempt.connection!, + state.outputBroadcast, + state.pipelineOutputsAudio, + state.pipelineOutputsVideo, + set + ) + ); + } +} + /** Core connection logic extracted from the store for reduced complexity. */ export async function performConnect( state: ConnectableState, decision: Extract, get: () => ConnectableState & { outputBroadcast: string }, - set: StateSetter + set: StateSetter, + abortSignal: AbortSignal ): Promise { const attempt: ConnectAttempt = { ...NULL_MOQ_REFS }; try { - logger.info('Step 1: Creating connection to relay server'); - const url = new URL(decision.trimmedServerUrl); - const jwt = state.moqToken.trim(); - if (jwt) { - url.searchParams.set('jwt', jwt); - } + if (abortSignal.aborted) throw new DOMException('Aborted', 'AbortError'); - attempt.connection = new Hang.Moq.Connection.Reload({ url, enabled: true }); - attempt.healthEffect = new Effect(); - setupConnectionStatusSync(attempt.healthEffect, attempt.connection, get, set); + const { connection, healthEffect } = createConnectionAndHealth( + decision.trimmedServerUrl, + state.moqToken, + get, + set + ); + attempt.connection = connection; + attempt.healthEffect = healthEffect; // Set up publish BEFORE watch. For external relay pipelines (pub/sub), // the skit pipeline needs input data before it can publish output. // If we watch first, the subscribe to output/catalog.json fails with // RESET_STREAM because skit hasn't started publishing yet. if (decision.shouldPublish) { + set({ connectingStep: 'devices' }); applyPublishResult( attempt, await setupPublishPath( @@ -533,40 +656,19 @@ export async function performConnect( state.inputBroadcast, state.pipelineNeedsAudio, state.pipelineNeedsVideo, - set + set, + abortSignal ) ); } - await waitForSignalValue( - attempt.connection.established, - (value) => value !== undefined, - 12_000, - 'Timed out connecting to MoQ gateway.' - ); + await connectWatchPath(attempt, state, decision, set, abortSignal); - if (decision.shouldWatch) { - // When publishing to an external relay, the skit pipeline needs time to - // discover input tracks, build the graph, and start publishing output. - // Wait for the output broadcast to be announced on the relay before - // subscribing, otherwise the catalog subscribe gets RESET_STREAM. - // In gateway mode the skit server manages the peer connection directly, - // so no announcement polling is needed. - if (decision.shouldPublish && state.isExternalRelay) { - await waitForBroadcastAnnouncement(attempt.connection, state.outputBroadcast); - } - - applyWatchResult( - attempt, - setupWatchPath( - attempt.healthEffect, - attempt.connection, - state.outputBroadcast, - state.pipelineOutputsAudio, - state.pipelineOutputsVideo, - set - ) - ); + // If aborted between the last await and now, discard this attempt + // so we don't overwrite a newer connect's state. + if (abortSignal.aborted) { + cleanupConnectAttempt(attempt); + return false; } schedulePostConnectWarnings(decision, attempt, get, set); @@ -574,6 +676,7 @@ export async function performConnect( set({ ...attempt, status: 'connected', + connectingStep: '', isMicEnabled: decision.shouldPublish && state.pipelineNeedsAudio, isCameraEnabled: decision.shouldPublish && state.pipelineNeedsVideo, }); @@ -584,11 +687,18 @@ export async function performConnect( logger.info(`Connection established: ${modes.join(' and ')}`); return true; } catch (error) { - logger.error('Connection failed:', error); cleanupConnectAttempt(attempt); + // If this attempt was aborted (superseded by disconnect or a newer + // connect), silently discard — don't overwrite the store. + if (abortSignal.aborted) { + return false; + } + + logger.error('Connection failed:', error); set({ status: 'disconnected', + connectingStep: '', watchStatus: 'disabled', micStatus: 'disabled', cameraStatus: 'disabled', diff --git a/ui/src/types/generated/api-types.ts b/ui/src/types/generated/api-types.ts index edac33f0..75fc5618 100644 --- a/ui/src/types/generated/api-types.ts +++ b/ui/src/types/generated/api-types.ts @@ -363,7 +363,12 @@ export type Node = { kind: string, params: JsonValue, */ state: NodeState | null, }; -export type Pipeline = { name: string | null, description: string | null, mode: EngineMode, nodes: Record, connections: Array, +export type Pipeline = { name: string | null, description: string | null, mode: EngineMode, +/** + * Declarative UI metadata — forwarded unchanged from `UserPipeline`, + * ignored by the engine for execution. + */ +client?: ClientSection | null, nodes: Record, connections: Array, /** * Resolved per-node view data (e.g., compositor layout). * Only populated in API responses; absent from pipeline definitions. @@ -418,4 +423,108 @@ export type ValidationError = { error_type: ValidationErrorType, message: string export type ValidationErrorType = "error" | "warning"; -export type PermissionsInfo = { create_sessions: boolean, destroy_sessions: boolean, list_sessions: boolean, modify_sessions: boolean, tune_nodes: boolean, load_plugins: boolean, delete_plugins: boolean, list_nodes: boolean, list_samples: boolean, read_samples: boolean, write_samples: boolean, delete_samples: boolean, access_all_sessions: boolean, upload_assets: boolean, delete_assets: boolean, }; \ No newline at end of file +export type PermissionsInfo = { create_sessions: boolean, destroy_sessions: boolean, list_sessions: boolean, modify_sessions: boolean, tune_nodes: boolean, load_plugins: boolean, delete_plugins: boolean, list_nodes: boolean, list_samples: boolean, read_samples: boolean, write_samples: boolean, delete_samples: boolean, access_all_sessions: boolean, upload_assets: boolean, delete_assets: boolean, }; + + +// client section +export type ClientSection = { +/** + * Direct relay URL for external MoQ relay pattern. + */ +relay_url: string | null, +/** + * Gateway path for gateway-managed MoQ pattern. + */ +gateway_path: string | null, +/** + * Browser-side publish configuration (dynamic pipelines). + */ +publish: PublishConfig | null, +/** + * Browser-side watch configuration (dynamic pipelines). + */ +watch: WatchConfig | null, +/** + * Input UX configuration (oneshot pipelines). + */ +input: InputConfig | null, +/** + * Output rendering configuration (oneshot pipelines). + */ +output: OutputConfig | null, }; + +export type PublishConfig = { +/** + * Broadcast name the browser publishes to. + */ +broadcast: string, +/** + * Whether the pipeline consumes audio from the browser. + */ +audio: boolean, +/** + * Whether the pipeline consumes video from the browser. + */ +video: boolean, }; + +export type WatchConfig = { +/** + * Broadcast name the browser subscribes to. + */ +broadcast: string, +/** + * Whether the pipeline outputs audio to subscribers. + */ +audio: boolean, +/** + * Whether the pipeline outputs video to subscribers. + */ +video: boolean, }; + +export type InputConfig = { +/** + * The kind of input UX to present. + */ +type: InputType, +/** + * MIME filter for file pickers (e.g. `audio/*`). + */ +accept: string | null, +/** + * Tags for filtering the asset picker (e.g. `["speech"]`). + */ +asset_tags: Array | null, +/** + * Placeholder text for text inputs. + */ +placeholder: string | null, +/** + * Per-field UI hints keyed by `http_input` field name. + */ +field_hints: Record | null, }; + +export type InputType = "file_upload" | "text" | "trigger" | "none"; + +export type OutputConfig = { +/** + * The media kind the pipeline produces. + */ +type: OutputType, }; + +export type OutputType = "transcription" | "json" | "audio" | "video"; + +export type FieldHint = { +/** + * Override the field's input type (default is file upload). + */ +type: FieldType | null, +/** + * MIME filter for file picker. + */ +accept: string | null, +/** + * Placeholder text for text inputs. + */ +placeholder: string | null, }; + +export type FieldType = "file" | "text"; \ No newline at end of file diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts new file mode 100644 index 00000000..d182e9c3 --- /dev/null +++ b/ui/src/utils/clientSection.test.ts @@ -0,0 +1,505 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +import { describe, expect, it } from 'vitest'; + +import type { ClientSection } from '@/types/types'; + +import { + extractClientFromParsed, + extractClientSection, + parseAcceptToFormats, + parseClientFromYaml, +} from './clientSection'; +import { deriveSettingsFromClient } from './moqPeerSettings'; + +describe('extractClientFromParsed', () => { + it('returns null for null input', () => { + expect(extractClientFromParsed(null)).toBeNull(); + }); + + it('returns null for undefined input', () => { + expect(extractClientFromParsed(undefined)).toBeNull(); + }); + + it('returns null when parsed object has no client key', () => { + expect(extractClientFromParsed({ nodes: {}, steps: [] })).toBeNull(); + }); + + it('extracts client section from a pre-parsed object', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/test', + publish: null, + watch: null, + input: { + type: 'file_upload', + accept: 'audio/opus', + asset_tags: null, + placeholder: null, + field_hints: null, + }, + output: { type: 'audio' }, + }; + const parsed = { nodes: {}, client }; + expect(extractClientFromParsed(parsed)).toBe(client); + }); +}); + +describe('extractClientSection', () => { + it('returns null for null pipeline', () => { + expect(extractClientSection(null)).toBeNull(); + }); + + it('returns null for undefined pipeline', () => { + expect(extractClientSection(undefined)).toBeNull(); + }); + + it('returns null when pipeline has no client', () => { + const pipeline = { nodes: {}, connections: [], mode: 'live' } as never; + expect(extractClientSection(pipeline)).toBeNull(); + }); + + it('returns client section when present', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/test', + publish: null, + watch: null, + input: null, + output: null, + }; + const pipeline = { nodes: {}, connections: [], mode: 'live', client } as never; + expect(extractClientSection(pipeline)).toBe(client); + }); +}); + +describe('deriveSettingsFromClient', () => { + it('derives settings for a gateway publish+watch pipeline', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/compositor', + publish: { broadcast: 'camera-feed', audio: true, video: true }, + watch: { broadcast: 'composited-output', audio: true, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: '/moq/compositor', + relayUrl: undefined, + inputBroadcast: 'camera-feed', + outputBroadcast: 'composited-output', + hasInputBroadcast: true, + needsAudioInput: true, + needsVideoInput: true, + outputsAudio: true, + outputsVideo: true, + }); + }); + + it('derives settings for a relay-based pipeline', () => { + const client: ClientSection = { + relay_url: 'https://relay.example.com', + gateway_path: null, + publish: { broadcast: 'input', audio: true, video: false }, + watch: { broadcast: 'output', audio: false, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: undefined, + relayUrl: 'https://relay.example.com', + inputBroadcast: 'input', + outputBroadcast: 'output', + hasInputBroadcast: true, + needsAudioInput: true, + needsVideoInput: false, + outputsAudio: false, + outputsVideo: true, + }); + }); + + it('derives settings for a watch-only pipeline', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/monitor', + publish: null, + watch: { broadcast: 'preview', audio: false, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: '/moq/monitor', + relayUrl: undefined, + inputBroadcast: undefined, + outputBroadcast: 'preview', + hasInputBroadcast: false, + needsAudioInput: false, + needsVideoInput: false, + outputsAudio: false, + outputsVideo: true, + }); + }); + + it('derives settings for a oneshot pipeline (no publish/watch)', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: null, + publish: null, + watch: null, + input: { + type: 'file_upload', + accept: 'audio/*', + asset_tags: null, + placeholder: null, + field_hints: null, + }, + output: { type: 'transcription' }, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: undefined, + relayUrl: undefined, + inputBroadcast: undefined, + outputBroadcast: undefined, + hasInputBroadcast: false, + needsAudioInput: false, + needsVideoInput: false, + outputsAudio: false, + outputsVideo: false, + }); + }); +}); + +describe('parseAcceptToFormats', () => { + it('returns null for null/undefined', () => { + expect(parseAcceptToFormats(null)).toBeNull(); + expect(parseAcceptToFormats(undefined)).toBeNull(); + }); + + it('returns null for empty string', () => { + expect(parseAcceptToFormats('')).toBeNull(); + }); + + it('returns null for wildcard accept', () => { + expect(parseAcceptToFormats('audio/*')).toBeNull(); + expect(parseAcceptToFormats('*/*')).toBeNull(); + }); + + it('parses dot-prefixed extensions', () => { + expect(parseAcceptToFormats('.ogg,.opus')).toEqual(['ogg', 'opus']); + }); + + it('parses MIME types', () => { + expect(parseAcceptToFormats('audio/wav,audio/ogg')).toEqual(['wav', 'ogg']); + }); + + it('parses mixed extensions and MIME types', () => { + expect(parseAcceptToFormats('audio/wav,.wav')).toEqual(['wav', 'wav']); + }); + + it('handles bare format names', () => { + expect(parseAcceptToFormats('ogg,opus')).toEqual(['ogg', 'opus']); + }); + + it('lowercases all formats', () => { + expect(parseAcceptToFormats('.OGG,.OPUS')).toEqual(['ogg', 'opus']); + }); + + it('trims whitespace', () => { + expect(parseAcceptToFormats(' .ogg , .opus ')).toEqual(['ogg', 'opus']); + }); +}); + +describe('parseClientFromYaml', () => { + it('returns null for empty YAML', () => { + expect(parseClientFromYaml('')).toBeNull(); + }); + + it('returns null for YAML without client section', () => { + const yaml = ` +steps: + - kind: ogg::demuxer +`; + expect(parseClientFromYaml(yaml)).toBeNull(); + }); + + it('parses a full client section from YAML', () => { + const yaml = ` +client: + input: + type: file_upload + accept: ".ogg,.opus" + asset_tags: + - speech + field_hints: + voice: + type: file + accept: "audio/wav,.wav" + output: + type: transcription +steps: + - kind: ogg::demuxer +`; + const client = parseClientFromYaml(yaml); + expect(client).not.toBeNull(); + expect(client?.input?.type).toBe('file_upload'); + expect(client?.input?.accept).toBe('.ogg,.opus'); + expect(client?.input?.asset_tags).toEqual(['speech']); + expect(client?.input?.field_hints?.voice?.type).toBe('file'); + expect(client?.input?.field_hints?.voice?.accept).toBe('audio/wav,.wav'); + expect(client?.output?.type).toBe('transcription'); + }); + + it('parses a dynamic pipeline client section', () => { + const yaml = ` +client: + gateway_path: /moq/compositor + publish: + broadcast: camera-feed + audio: true + video: true + watch: + broadcast: composited-output + audio: true + video: true +nodes: + pub: + kind: moq::subscriber +`; + const client = parseClientFromYaml(yaml); + expect(client).not.toBeNull(); + expect(client?.gateway_path).toBe('/moq/compositor'); + expect(client?.publish?.broadcast).toBe('camera-feed'); + expect(client?.watch?.broadcast).toBe('composited-output'); + }); + + it('returns null for invalid YAML', () => { + expect(parseClientFromYaml('{{invalid')).toBeNull(); + }); +}); + +// ----------------------------------------------------------------------- +// Monitor preview config derivation (Gap 4a) +// ----------------------------------------------------------------------- + +describe('deriveSettingsFromClient — monitor preview scenarios', () => { + it('derives watch-only settings for monitor preview (no publish)', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/transcoder', + publish: null, + watch: { broadcast: 'output', audio: true, video: false }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings.gatewayPath).toBe('/moq/transcoder'); + expect(settings.outputBroadcast).toBe('output'); + expect(settings.hasInputBroadcast).toBe(false); + expect(settings.outputsAudio).toBe(true); + expect(settings.outputsVideo).toBe(false); + }); + + it('derives audio+video output for compositor preview', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/compositor', + publish: { broadcast: 'camera', audio: true, video: true }, + watch: { broadcast: 'composited', audio: true, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings.outputsAudio).toBe(true); + expect(settings.outputsVideo).toBe(true); + expect(settings.outputBroadcast).toBe('composited'); + }); + + it('derives relay-url settings for external relay preview', () => { + const client: ClientSection = { + relay_url: 'https://relay.example.com', + gateway_path: null, + publish: null, + watch: { broadcast: 'preview', audio: false, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings.relayUrl).toBe('https://relay.example.com'); + expect(settings.gatewayPath).toBeUndefined(); + expect(settings.outputsVideo).toBe(true); + expect(settings.outputsAudio).toBe(false); + }); +}); + +// ----------------------------------------------------------------------- +// ConvertView-driven pipeline classification (Gap 4b) +// ----------------------------------------------------------------------- + +describe('parseClientFromYaml — trigger vs none and output-type', () => { + it('parses trigger input type', () => { + const yaml = ` +client: + input: + type: trigger + output: + type: video +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + expect(client).not.toBeNull(); + expect(client?.input?.type).toBe('trigger'); + expect(client?.output?.type).toBe('video'); + }); + + it('parses none input type', () => { + const yaml = ` +client: + input: + type: none + output: + type: video +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + expect(client?.input?.type).toBe('none'); + }); + + it('identifies transcription output type for renderer selection', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: transcription +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('transcription'); + // ConvertView uses: isTranscriptionPipeline = client?.output?.type === 'transcription' + expect(client?.output?.type === 'transcription').toBe(true); + }); + + it('identifies audio output type for audio player renderer', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('audio'); + // ConvertView renders CustomAudioPlayer for audio output + expect(client?.output?.type === 'audio').toBe(true); + }); + + it('identifies video output type for video player renderer', () => { + const yaml = ` +client: + input: + type: none + output: + type: video +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('video'); + // ConvertView: isVideoPipeline = client?.output?.type === 'video' + const isNoInput = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + expect(isNoInput).toBe(true); + }); + + it('identifies json output type for JsonStreamDisplay', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: json +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('json'); + }); + + it('classifies text input as TTS pipeline', () => { + const yaml = ` +client: + input: + type: text + placeholder: "Enter text to convert to speech" + output: + type: audio +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + // ConvertView: isTTSPipeline = client?.input?.type === 'text' + expect(client?.input?.type).toBe('text'); + expect(client?.input?.type === 'text').toBe(true); + }); + + it('trigger and none both classify as no-input pipeline', () => { + for (const inputType of ['trigger', 'none']) { + const yaml = ` +client: + input: + type: ${inputType} + output: + type: audio +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + // ConvertView: isNoInputPipeline = client?.input?.type === 'none' || client?.input?.type === 'trigger' + const isNoInput = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + expect(isNoInput).toBe(true); + } + }); + + it('file_upload does NOT classify as no-input pipeline', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + const isNoInput = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + expect(isNoInput).toBe(false); + }); +}); diff --git a/ui/src/utils/clientSection.ts b/ui/src/utils/clientSection.ts new file mode 100644 index 00000000..6ff2273e --- /dev/null +++ b/ui/src/utils/clientSection.ts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +import { load } from 'js-yaml'; + +import type { ClientSection, Pipeline } from '@/types/types'; + +/** + * Extracts the `client` section from a compiled `Pipeline`, returning `null` + * when the field is absent. + */ +export function extractClientSection(pipeline: Pipeline | null | undefined): ClientSection | null { + return pipeline?.client ?? null; +} + +/** + * Extracts the `client` section from an already-parsed YAML object. + * Use this when you have already called `load()` and want to avoid + * parsing the same YAML string again. + */ +export function extractClientFromParsed( + parsed: Record | null | undefined +): ClientSection | null { + if (!parsed || typeof parsed !== 'object') return null; + return (parsed.client as ClientSection) ?? null; +} + +/** + * Parses a raw pipeline YAML string and extracts the `client` section. + * Returns `null` if the YAML is invalid or has no client section. + */ +export function parseClientFromYaml(yamlContent: string): ClientSection | null { + try { + const parsed = load(yamlContent) as Record | null; + return extractClientFromParsed(parsed); + } catch { + return null; + } +} + +/** + * Converts a CSS-style `accept` attribute (e.g. `"audio/*"`, `".ogg,.opus"`) + * into an array of lowercase format names suitable for asset-format matching. + * Returns `null` when all formats are acceptable. + */ +export function parseAcceptToFormats(accept: string | null | undefined): string[] | null { + if (!accept) return null; + if (accept.includes('*')) return null; + + const formats: string[] = []; + for (const part of accept.split(',')) { + const trimmed = part.trim(); + if (!trimmed) continue; + if (trimmed.startsWith('.')) { + formats.push(trimmed.slice(1).toLowerCase()); + } else if (trimmed.includes('/')) { + const sub = trimmed.split('/')[1]?.toLowerCase(); + if (sub) formats.push(sub); + } else { + formats.push(trimmed.toLowerCase()); + } + } + return formats.length > 0 ? formats : null; +} diff --git a/ui/src/utils/moqPeerSettings.test.ts b/ui/src/utils/moqPeerSettings.test.ts index e49678a8..1091841b 100644 --- a/ui/src/utils/moqPeerSettings.test.ts +++ b/ui/src/utils/moqPeerSettings.test.ts @@ -4,10 +4,10 @@ import { describe, expect, it } from 'vitest'; -import { extractMoqPeerSettings } from './moqPeerSettings'; +import { extractMoqPeerSettings, updateUrlPath } from './moqPeerSettings'; // --------------------------------------------------------------------------- -// extractMoqPeerSettings — basic parsing +// extractMoqPeerSettings — reads the declarative `client` section // --------------------------------------------------------------------------- describe('extractMoqPeerSettings', () => { @@ -15,311 +15,209 @@ describe('extractMoqPeerSettings', () => { expect(extractMoqPeerSettings('')).toBeNull(); }); - it('should return null for YAML without nodes', () => { + it('should return null for YAML without client section', () => { expect(extractMoqPeerSettings('name: test')).toBeNull(); }); - it('should return null when no moq_peer node exists', () => { - const yaml = ` -nodes: - gain: - kind: audio::gain - params: - gain: 1.0 -`; - expect(extractMoqPeerSettings(yaml)).toBeNull(); - }); - it('should return null for invalid YAML', () => { expect(extractMoqPeerSettings('{{{')).toBeNull(); }); - it('should return null when moq_peer has no params', () => { + it('should return null for oneshot client (input/output only)', () => { const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio nodes: - moq_peer: - kind: transport::moq::peer + gain: + kind: audio::gain `; expect(extractMoqPeerSettings(yaml)).toBeNull(); }); - it('should extract basic moq_peer settings', () => { + it('should extract gateway-based settings with publish and watch', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output +client: + gateway_path: /moq/echo + publish: + broadcast: echo-demo + audio: true + video: false + watch: + broadcast: echo-demo + audio: true + video: false `; const result = extractMoqPeerSettings(yaml); expect(result).not.toBeNull(); - expect(result!.gatewayPath).toBe('/moq'); - expect(result!.inputBroadcast).toBe('input'); - expect(result!.outputBroadcast).toBe('output'); + expect(result!.gatewayPath).toBe('/moq/echo'); + expect(result!.inputBroadcast).toBe('echo-demo'); + expect(result!.outputBroadcast).toBe('echo-demo'); expect(result!.hasInputBroadcast).toBe(true); - }); - - it('should report hasInputBroadcast false when input_broadcast is absent', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output -`; - const result = extractMoqPeerSettings(yaml); - expect(result).not.toBeNull(); - expect(result!.hasInputBroadcast).toBe(false); - }); -}); - -// --------------------------------------------------------------------------- -// detectPeerInputMediaTypes (exercised through extractMoqPeerSettings) -// --------------------------------------------------------------------------- - -describe('detectPeerInputMediaTypes (via extractMoqPeerSettings)', () => { - it('should detect audio input (bare peer reference)', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - opus_decoder: - kind: audio::opus::decoder - needs: moq_peer -`; - const result = extractMoqPeerSettings(yaml); expect(result!.needsAudioInput).toBe(true); expect(result!.needsVideoInput).toBe(false); + expect(result!.outputsAudio).toBe(true); + expect(result!.outputsVideo).toBe(false); }); - it('should detect audio input (explicit .out pin)', () => { + it('should extract relay-based settings', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - opus_decoder: - kind: audio::opus::decoder - needs: moq_peer.out +client: + relay_url: "https://relay.example.com" + publish: + broadcast: input-stream + audio: true + video: true + watch: + broadcast: output-stream + audio: true + video: false `; const result = extractMoqPeerSettings(yaml); + expect(result).not.toBeNull(); + expect(result!.relayUrl).toBe('https://relay.example.com'); + expect(result!.gatewayPath).toBeUndefined(); + expect(result!.inputBroadcast).toBe('input-stream'); + expect(result!.outputBroadcast).toBe('output-stream'); + expect(result!.hasInputBroadcast).toBe(true); expect(result!.needsAudioInput).toBe(true); - expect(result!.needsVideoInput).toBe(false); + expect(result!.needsVideoInput).toBe(true); + expect(result!.outputsAudio).toBe(true); + expect(result!.outputsVideo).toBe(false); }); - it('should detect video input (.out_1 pin)', () => { + it('should report hasInputBroadcast false when publish is absent', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - vp9_decoder: - kind: video::vp9::decoder - needs: moq_peer.out_1 +client: + gateway_path: /moq/colorbars + watch: + broadcast: colorbars + audio: false + video: true `; const result = extractMoqPeerSettings(yaml); + expect(result).not.toBeNull(); + expect(result!.hasInputBroadcast).toBe(false); expect(result!.needsAudioInput).toBe(false); - expect(result!.needsVideoInput).toBe(true); + expect(result!.needsVideoInput).toBe(false); + expect(result!.outputsAudio).toBe(false); + expect(result!.outputsVideo).toBe(true); }); - it('should detect both audio and video inputs', () => { + it('should detect audio+video publish and watch', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - opus_decoder: - kind: audio::opus::decoder - needs: moq_peer.out - vp9_decoder: - kind: video::vp9::decoder - needs: moq_peer.out_1 +client: + gateway_path: /moq/av + publish: + broadcast: av-input + audio: true + video: true + watch: + broadcast: av-output + audio: true + video: true `; const result = extractMoqPeerSettings(yaml); expect(result!.needsAudioInput).toBe(true); expect(result!.needsVideoInput).toBe(true); + expect(result!.outputsAudio).toBe(true); + expect(result!.outputsVideo).toBe(true); }); - it('should handle needs as array', () => { + it('should handle watch-only pipeline (no publish)', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - mixer: - kind: audio::mixer - needs: - - moq_peer - - other_source +client: + gateway_path: /moq/output + watch: + broadcast: output-stream + audio: true + video: true `; const result = extractMoqPeerSettings(yaml); - expect(result!.needsAudioInput).toBe(true); + expect(result).not.toBeNull(); + expect(result!.hasInputBroadcast).toBe(false); + expect(result!.needsAudioInput).toBe(false); + expect(result!.needsVideoInput).toBe(false); + expect(result!.outputsAudio).toBe(true); + expect(result!.outputsVideo).toBe(true); }); - it('should handle needs as map (record)', () => { + it('should return settings when only gateway_path is present', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - compositor: - kind: video::compositor - needs: - in: colorbars - in_1: moq_peer.out_1 +client: + gateway_path: /moq/peer `; const result = extractMoqPeerSettings(yaml); - expect(result!.needsVideoInput).toBe(true); + expect(result).not.toBeNull(); + expect(result!.gatewayPath).toBe('/moq/peer'); + expect(result!.hasInputBroadcast).toBe(false); + expect(result!.outputsAudio).toBe(false); + expect(result!.outputsVideo).toBe(false); }); - it('should report no inputs when no downstream nodes reference the peer', () => { + it('should return settings when only relay_url is present', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - gain: - kind: audio::gain - needs: some_other_node +client: + relay_url: "https://relay.example.com" `; const result = extractMoqPeerSettings(yaml); - expect(result!.needsAudioInput).toBe(false); - expect(result!.needsVideoInput).toBe(false); + expect(result).not.toBeNull(); + expect(result!.relayUrl).toBe('https://relay.example.com'); + expect(result!.hasInputBroadcast).toBe(false); }); -}); -// --------------------------------------------------------------------------- -// detectPeerOutputMediaTypes (exercised through extractMoqPeerSettings) -// --------------------------------------------------------------------------- - -describe('detectPeerOutputMediaTypes (via extractMoqPeerSettings)', () => { - it('should detect audio output when moq_peer needs an audio:: node', () => { + it('should handle audio-only publish', () => { const yaml = ` -nodes: - opus_encoder: - kind: audio::opus::encoder - needs: gain - gain: - kind: audio::gain - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: opus_encoder +client: + gateway_path: /moq/audio + publish: + broadcast: mic + audio: true + video: false + watch: + broadcast: processed + audio: true + video: false `; const result = extractMoqPeerSettings(yaml); + expect(result!.needsAudioInput).toBe(true); + expect(result!.needsVideoInput).toBe(false); expect(result!.outputsAudio).toBe(true); expect(result!.outputsVideo).toBe(false); }); +}); - it('should detect video output when moq_peer needs a video:: node', () => { - const yaml = ` -nodes: - vp9_encoder: - kind: video::vp9::encoder - needs: pixel_convert - pixel_convert: - kind: video::pixel_convert - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: vp9_encoder -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.outputsAudio).toBe(false); - expect(result!.outputsVideo).toBe(true); - }); +// --------------------------------------------------------------------------- +// updateUrlPath — preserves host when applying a gateway path +// --------------------------------------------------------------------------- - it('should detect both audio and video outputs', () => { - const yaml = ` -nodes: - opus_encoder: - kind: audio::opus::encoder - vp9_encoder: - kind: video::vp9::encoder - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: - in: opus_encoder - in_1: vp9_encoder -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.outputsAudio).toBe(true); - expect(result!.outputsVideo).toBe(true); +describe('updateUrlPath', () => { + it('should replace path on a standard URL', () => { + expect(updateUrlPath('http://127.0.0.1:4545/moq', '/moq/echo')).toBe( + 'http://127.0.0.1:4545/moq/echo' + ); }); - it('should report no outputs when moq_peer has no needs', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.outputsAudio).toBe(false); - expect(result!.outputsVideo).toBe(false); + it('should replace path on a relay URL (regression: gateway→relay→gateway)', () => { + // If the caller mistakenly passes a relay URL as baseUrl when switching + // back to a gateway pipeline, the result keeps the relay host — which is + // the bug. The fix is for the *caller* to always pass the original + // config URL, but this test documents updateUrlPath's expected behaviour. + expect(updateUrlPath('http://localhost:4443', '/moq')).toBe('http://localhost:4443/moq'); }); - it('should handle dotted pin references in moq_peer needs', () => { - const yaml = ` -nodes: - opus_encoder: - kind: audio::opus::encoder - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: opus_encoder.out -`; - const result = extractMoqPeerSettings(yaml); - // "opus_encoder.out" → nodeName is "opus_encoder" → kind is audio:: → outputsAudio - expect(result!.outputsAudio).toBe(true); + it('should handle URLs with trailing slashes', () => { + expect(updateUrlPath('http://example.com:4545/', '/moq/transcoder')).toBe( + 'http://example.com:4545/moq/transcoder' + ); }); - it('should ignore upstream nodes without a kind', () => { - const yaml = ` -nodes: - unknown_node: {} - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - needs: unknown_node -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.outputsAudio).toBe(false); - expect(result!.outputsVideo).toBe(false); + it('should preserve protocol and port', () => { + expect(updateUrlPath('https://host.example.com:9443/old-path', '/moq/new')).toBe( + 'https://host.example.com:9443/moq/new' + ); }); }); diff --git a/ui/src/utils/moqPeerSettings.ts b/ui/src/utils/moqPeerSettings.ts index 99ec803d..be062c61 100644 --- a/ui/src/utils/moqPeerSettings.ts +++ b/ui/src/utils/moqPeerSettings.ts @@ -2,7 +2,9 @@ // // SPDX-License-Identifier: MPL-2.0 -import { load } from 'js-yaml'; +import type { ClientSection } from '@/types/types'; + +import { parseClientFromYaml } from './clientSection'; export interface MoqPeerSettings { gatewayPath?: string; @@ -22,225 +24,44 @@ export interface MoqPeerSettings { outputsVideo: boolean; } -type NeedsValue = string | string[] | Record; - -type ParsedNode = { - kind?: string; - params?: { - gateway_path?: string; - input_broadcast?: string; - output_broadcast?: string; - url?: string; - broadcast?: string; - audio?: boolean; - video?: boolean; - [key: string]: unknown; - }; - needs?: NeedsValue; -}; - -type ParsedYaml = { - nodes?: Record; -}; - /** - * Collects all dependency references from a node's `needs` field as flat strings. + * Derives `MoqPeerSettings` from a declarative `ClientSection`. */ -function collectNeedsRefs(needs: NeedsValue | undefined): string[] { - if (!needs) return []; - if (typeof needs === 'string') return [needs]; - if (Array.isArray(needs)) return needs.filter((v): v is string => typeof v === 'string'); - // Record (map variant) — values are the dependency refs - return Object.values(needs).filter((v): v is string => typeof v === 'string'); -} - -/** - * Scans all nodes in the pipeline to detect which media types downstream nodes - * consume from the moq_peer's output pins. - * - * The `transport::moq::peer` node exposes two fixed output pins: - * - `out` → audio (Opus-encoded) - * - `out_1` → video (VP9-encoded) - * - * This is a stable naming convention enforced by the backend node - * definition (see `MoqPeerNode::output_pins` in `crates/nodes/src/transport/moq/peer.rs`). - * A bare reference to the peer name (e.g. `moq_peer`) is equivalent to `moq_peer.out`. - */ -function detectPeerInputMediaTypes( - peerName: string, - nodes: Record -): { needsAudio: boolean; needsVideo: boolean } { - let needsAudio = false; - let needsVideo = false; - - for (const [nodeName, nodeConfig] of Object.entries(nodes)) { - if (nodeName === peerName) continue; - for (const ref of collectNeedsRefs(nodeConfig.needs)) { - if (ref === peerName || ref === `${peerName}.out`) { - needsAudio = true; - } else if (ref === `${peerName}.out_1`) { - needsVideo = true; - } - } - } - - return { needsAudio, needsVideo }; -} - -/** - * Detects what media types the moq_peer outputs to subscribers by looking at - * which upstream nodes are connected to its inputs. The node `kind` prefix - * (`audio::` or `video::`) determines the media type. - */ -function detectPeerOutputMediaTypes( - peerName: string, - nodes: Record -): { outputsAudio: boolean; outputsVideo: boolean } { - const peerNode = nodes[peerName]; - if (!peerNode) return { outputsAudio: false, outputsVideo: false }; - - let outputsAudio = false; - let outputsVideo = false; - - for (const ref of collectNeedsRefs(peerNode.needs)) { - const nodeName = ref.split('.')[0]; - const sourceNode = nodes[nodeName]; - if (!sourceNode?.kind) continue; - - if (sourceNode.kind.startsWith('audio::')) { - outputsAudio = true; - } else if (sourceNode.kind.startsWith('video::')) { - outputsVideo = true; - } - } - - return { outputsAudio, outputsVideo }; -} - -/** Finds the first subscriber and publisher nodes in the pipeline. */ -function findPubSubNodes(nodes: Record): { - subscriberName: string | null; - subscriberConfig: ParsedNode | null; - publisherConfig: ParsedNode | null; -} { - let subscriberName: string | null = null; - let subscriberConfig: ParsedNode | null = null; - let publisherConfig: ParsedNode | null = null; - - for (const [name, nodeConfig] of Object.entries(nodes)) { - if (nodeConfig.kind === 'transport::moq::subscriber') { - subscriberName = name; - subscriberConfig = nodeConfig; - } else if (nodeConfig.kind === 'transport::moq::publisher') { - publisherConfig = nodeConfig; - } - } - - return { subscriberName, subscriberConfig, publisherConfig }; -} - -/** - * Detects which media types downstream nodes consume from a subscriber node - * by scanning `needs` references across all nodes. - */ -function detectSubscriberInputMediaTypes( - subscriberName: string, - nodes: Record -): { needsAudio: boolean; needsVideo: boolean } { - let needsAudio = false; - let needsVideo = false; - - for (const nodeConfig of Object.values(nodes)) { - for (const ref of collectNeedsRefs(nodeConfig.needs)) { - if (ref === subscriberName || ref === `${subscriberName}.out`) { - needsAudio = true; - } else if (ref.startsWith(`${subscriberName}.`) && ref.includes('video')) { - needsVideo = true; - } - } - } - - return { needsAudio, needsVideo }; -} - -/** - * Detects media types for pipelines using separate `transport::moq::publisher` - * and `transport::moq::subscriber` nodes (external relay pattern). - */ -function extractPubSubSettings(nodes: Record): MoqPeerSettings | null { - const { subscriberName, subscriberConfig, publisherConfig } = findPubSubNodes(nodes); - if (!subscriberConfig && !publisherConfig) return null; - - const subParams = subscriberConfig?.params; - const pubParams = publisherConfig?.params; - - const inputMedia = - subscriberName != null - ? detectSubscriberInputMediaTypes(subscriberName, nodes) - : { needsAudio: false, needsVideo: false }; - +export function deriveSettingsFromClient(client: ClientSection): MoqPeerSettings { return { - relayUrl: subParams?.url ?? pubParams?.url, - inputBroadcast: subParams?.broadcast, - outputBroadcast: pubParams?.broadcast, - hasInputBroadcast: Boolean(subParams?.broadcast), - needsAudioInput: inputMedia.needsAudio, - needsVideoInput: inputMedia.needsVideo, - outputsAudio: pubParams?.audio === true, - outputsVideo: pubParams?.video === true, + gatewayPath: client.gateway_path ?? undefined, + relayUrl: client.relay_url ?? undefined, + inputBroadcast: client.publish?.broadcast, + outputBroadcast: client.watch?.broadcast, + hasInputBroadcast: Boolean(client.publish), + needsAudioInput: client.publish?.audio ?? false, + needsVideoInput: client.publish?.video ?? false, + outputsAudio: client.watch?.audio ?? false, + outputsVideo: client.watch?.video ?? false, }; } /** - * Extracts moq_peer settings from a pipeline YAML string. - * Looks for `transport::moq::peer` nodes first (gateway pattern), then falls - * back to separate `transport::moq::publisher`/`subscriber` nodes (external - * relay pattern). + * Extracts MoQ peer settings from a pipeline YAML string by reading the + * declarative `client` section. + * + * Returns settings only when the client section declares dynamic transport + * configuration (gateway_path, relay_url, publish, or watch). Oneshot + * pipelines (input/output only) return null. * * @param yamlContent - The YAML string to parse - * @returns MoqPeerSettings if MoQ transport nodes are found, null otherwise + * @returns MoqPeerSettings if the client section declares MoQ transport, null otherwise */ export function extractMoqPeerSettings(yamlContent: string): MoqPeerSettings | null { - try { - const parsed = load(yamlContent) as ParsedYaml; - - if (!parsed || typeof parsed !== 'object' || !parsed.nodes) { - return null; - } + const client = parseClientFromYaml(yamlContent); + if (!client) return null; - // Find the first node with kind 'transport::moq::peer' - let peerNodeName: string | null = null; - let peerNodeConfig: ParsedNode | null = null; - for (const [name, nodeConfig] of Object.entries(parsed.nodes)) { - if (nodeConfig.kind === 'transport::moq::peer') { - peerNodeName = name; - peerNodeConfig = nodeConfig; - break; - } - } - - // Gateway pattern: transport::moq::peer - if (peerNodeName && peerNodeConfig?.params) { - const { needsAudio, needsVideo } = detectPeerInputMediaTypes(peerNodeName, parsed.nodes); - const { outputsAudio, outputsVideo } = detectPeerOutputMediaTypes(peerNodeName, parsed.nodes); - - return { - gatewayPath: peerNodeConfig.params.gateway_path, - inputBroadcast: peerNodeConfig.params.input_broadcast, - outputBroadcast: peerNodeConfig.params.output_broadcast, - hasInputBroadcast: Boolean(peerNodeConfig.params.input_broadcast), - needsAudioInput: needsAudio, - needsVideoInput: needsVideo, - outputsAudio, - outputsVideo, - }; - } - - // External relay pattern: separate publisher/subscriber nodes - return extractPubSubSettings(parsed.nodes); - } catch { + // Only return settings for dynamic pipelines that declare MoQ transport. + if (!client.gateway_path && !client.relay_url && !client.publish && !client.watch) { return null; } + + return deriveSettingsFromClient(client); } /** diff --git a/ui/src/views/ConvertView.tsx b/ui/src/views/ConvertView.tsx index e4e208c2..547be603 100644 --- a/ui/src/views/ConvertView.tsx +++ b/ui/src/views/ConvertView.tsx @@ -39,12 +39,16 @@ import { } from '@/services/converter'; import { listSamples } from '@/services/samples'; import { ensureSchemasLoaded, useSchemaStore } from '@/stores/schemaStore'; +import { + extractClientFromParsed, + parseAcceptToFormats, + parseClientFromYaml, +} from '@/utils/clientSection'; import { viewsLogger } from '@/utils/logger'; import { orderSamplePipelinesSystemFirst } from '@/utils/samplePipelineOrdering'; import { injectFileReadNode } from '@/utils/yamlPipeline'; type HttpInputField = { name: string; required: boolean }; -type InputFormatSpec = { all: string[]; perField: Record }; const resolveUploadFields = (httpInputFields: HttpInputField[]): HttpInputField[] => httpInputFields.length > 0 ? httpInputFields : [{ name: 'media', required: true }]; @@ -89,11 +93,10 @@ const extractFieldsFromNode = ( return fallback ? [{ name: fallback, required: defaultField ? false : true }] : []; }; -const deriveHttpInputFields = ( - yaml: string +const deriveHttpInputFieldsFromParsed = ( + parsed: Record | null | undefined ): { fields: HttpInputField[]; hasHttpInput: boolean } => { try { - const parsed = loadYaml(yaml) as { nodes?: unknown; steps?: unknown } | null; if (!parsed || typeof parsed !== 'object') return { fields: [], hasHttpInput: false }; if (isRecord(parsed.nodes)) { @@ -360,91 +363,6 @@ const CharCounter = styled.div` text-align: right; `; -// Helper functions moved outside component (pure functions, no dependencies) - -/** - * Detects if the current pipeline is a transcription pipeline - */ -const checkIfTranscriptionPipeline = (yaml: string): boolean => { - // A transcription pipeline is one that produces `Transcription` packets. - // `core::json_serialize` is used by many pipelines (VAD events, etc.) so it is not a signal. - const lowerYaml = yaml.toLowerCase(); - return ( - lowerYaml.includes('plugin::native::whisper') || - lowerYaml.includes('plugin::native::sensevoice') || - lowerYaml.includes('transcription') - ); -}; - -/** - * Detects if the current pipeline generates its own input (no user input needed) - */ -const checkIfNoInputPipeline = (yaml: string): boolean => { - const lowerYaml = yaml.toLowerCase(); - - // Check if pipeline starts with a script node that uses fetch() - // This indicates the pipeline generates its own data - if (lowerYaml.includes('core::script') && lowerYaml.includes('fetch')) { - return true; - } - - // Pipelines that have http_output but no http_input are self-contained generators - // (e.g. video::colorbars → encoder → muxer → http_output) - if ( - lowerYaml.includes('streamkit::http_output') && - !lowerYaml.includes('streamkit::http_input') - ) { - return true; - } - - return false; -}; - -/** - * Detects if the current pipeline is a TTS pipeline (text input) - */ -const checkIfTTSPipeline = (yaml: string): boolean => { - // First check if it's a no-input pipeline (takes precedence) - if (checkIfNoInputPipeline(yaml)) { - return false; - } - - // A TTS pipeline for text input should have text_chunker as an early node - // Just having TTS nodes isn't enough - the pipeline might use TTS as a component - // in a larger audio-to-audio pipeline (like speech translation) - const lowerYaml = yaml.toLowerCase(); - - // Check for text_chunker which indicates text input processing - if (lowerYaml.includes('text_chunker')) { - return true; - } - - // Additional heuristic: If we have TTS but NO audio demuxers/decoders, - // it's likely a text input pipeline - const hasTTS = - lowerYaml.includes('kokoro_tts') || - lowerYaml.includes('piper_tts') || - lowerYaml.includes('text-to-speech'); - - const hasAudioDemuxer = lowerYaml.includes('demux') || lowerYaml.includes('decode'); - - // If we have TTS but no audio demuxer, it's a text input pipeline - return hasTTS && !hasAudioDemuxer; -}; - -/** - * Detects if the current pipeline produces video output. - * Checks for `video::` node kind prefixes, and `video_width` / `video_height` - * when they appear as YAML mapping keys (not inside comments or arbitrary strings). - */ -const checkIfVideoPipeline = (yaml: string): boolean => { - const lowerYaml = yaml.toLowerCase(); - if (lowerYaml.includes('video::')) return true; - // Match video_width / video_height only as YAML keys (leading whitespace + colon suffix) - // to avoid false-positives on comments or unrelated string values. - return /^\s*video_width\s*:/m.test(lowerYaml) || /^\s*video_height\s*:/m.test(lowerYaml); -}; - const resolveTextField = (fields: HttpInputField[]): HttpInputField | null => { if (fields.length === 0) { return null; @@ -466,77 +384,6 @@ const splitTtsFields = ( }; }; -const FORMAT_ACCEPT_MAP: Record = { - ogg: '.ogg', - opus: '.opus', - mp3: '.mp3', - wav: '.wav', - flac: '.flac', - txt: '.txt', - text: '.txt', - json: '.json', -}; - -const detectInputFormatSpec = (yaml: string): InputFormatSpec | null => { - const match = yaml.match(/^\s*#\s*skit:input_formats\s*=\s*([^\n#]+)\s*$/im); - if (!match?.[1]) return null; - - const spec: InputFormatSpec = { all: [], perField: {} }; - - for (const entry of match[1].split(',')) { - const token = entry.trim(); - if (!token) continue; - - const parts = token.split(':'); - if (parts.length > 1) { - const field = parts[0].trim().toLowerCase(); - const formats = parts - .slice(1) - .join(':') - .split(/[|+]/) - .map((format) => format.trim().toLowerCase()) - .filter(Boolean); - - if (!field || formats.length === 0) continue; - - const existing = spec.perField[field] ?? []; - for (const format of formats) { - if (!existing.includes(format)) { - existing.push(format); - } - } - spec.perField[field] = existing; - } else { - const format = token.toLowerCase(); - if (!spec.all.includes(format)) { - spec.all.push(format); - } - } - } - - if (spec.all.length === 0 && Object.keys(spec.perField).length === 0) { - return null; - } - - return spec; -}; - -const resolveFormatsForField = ( - fieldName: string, - formatSpec: InputFormatSpec | null, - fallbackFormats: string[] | null -): string[] | null => { - const key = fieldName.toLowerCase(); - const perField = formatSpec?.perField?.[key]; - if (perField && perField.length > 0) { - return perField; - } - if (formatSpec?.all && formatSpec.all.length > 0) { - return formatSpec.all; - } - return fallbackFormats; -}; - const buildNoInputUploads = (fields: HttpInputField[]): UploadField[] => { // Generator pipelines (e.g. video::colorbars) have no http_input at all — // return empty so the multipart request only contains the config field. @@ -603,37 +450,12 @@ const buildUploadModeUploads = ( return [{ field: fields[0].name, file: selectedFile }]; }; -const formatHintForField = ( - field: HttpInputField, - formatSpec: InputFormatSpec | null, - fallbackFormats: string[] | null, - isTts: boolean -): { accept?: string; hint?: string } => { - const name = field.name.toLowerCase(); - const fieldOverrides = - (formatSpec?.all?.length ?? 0) > 0 || (formatSpec?.perField?.[name]?.length ?? 0) > 0; - - if (isTts && name.includes('voice') && !fieldOverrides) { - return { accept: 'audio/wav,.wav,.wave', hint: 'Expected format: WAV audio' }; - } - - const formats = resolveFormatsForField(field.name, formatSpec, fallbackFormats); - if (!formats || formats.length === 0) { - return {}; - } - - const unique = Array.from(new Set(formats.map((format) => format.toLowerCase()))); - const accept = unique - .map((format) => FORMAT_ACCEPT_MAP[format]) - .filter(Boolean) - .join(','); - const label = unique.map((format) => format.toUpperCase()).join(', '); - const hint = `Expected format${unique.length > 1 ? 's' : ''}: ${label}`; - - return { - accept: accept || undefined, - hint, - }; +const assetMatchesTag = (assetId: string, tag: string): boolean => { + if (tag === 'speech') return assetId.toLowerCase().startsWith('speech_'); + if (tag === 'music') return assetId.toLowerCase().startsWith('music_'); + if (tag.startsWith('id:')) + return assetId.toLowerCase() === tag.slice('id:'.length).trim().toLowerCase(); + return false; }; /** @@ -731,12 +553,6 @@ const ConvertView: React.FC = () => { setPipelineYaml, selectedTemplateId, setSelectedTemplateId, - isTranscriptionPipeline, - setIsTranscriptionPipeline, - isTTSPipeline, - setIsTTSPipeline, - isNoInputPipeline, - setIsNoInputPipeline, textInput, setTextInput, conversionStatus, @@ -769,8 +585,22 @@ const ConvertView: React.FC = () => { const [msePlaybackError, setMsePlaybackError] = useState(null); const [mseFallbackLoading, setMseFallbackLoading] = useState(false); - // Derived: detect if the selected pipeline produces video output - const isVideoPipeline = useMemo(() => checkIfVideoPipeline(pipelineYaml), [pipelineYaml]); + // Parse pipeline YAML once — derive both client section and http_input fields + // from the same parsed object to avoid double-parsing. + const parsedPipelineYaml = useMemo(() => { + try { + const parsed = loadYaml(pipelineYaml) as Record | null; + return parsed && typeof parsed === 'object' ? parsed : null; + } catch { + return null; + } + }, [pipelineYaml]); + + const client = useMemo(() => extractClientFromParsed(parsedPipelineYaml), [parsedPipelineYaml]); + const isTranscriptionPipeline = client?.output?.type === 'transcription'; + const isTTSPipeline = client?.input?.type === 'text'; + const isNoInputPipeline = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + const isVideoPipeline = client?.output?.type === 'video'; // Generate CLI command based on current template and pipeline type const cliCommand = useMemo(() => { @@ -823,101 +653,39 @@ const ConvertView: React.FC = () => { // Fetch audio assets const { data: audioAssets = [], isLoading: assetsLoading } = useAudioAssets(); - /** - * Detects the expected input format(s) from a pipeline YAML - * Returns an array of compatible formats, or null if any format is acceptable - */ - const detectExpectedFormats = (yaml: string): string[] | null => { - const lowerYaml = yaml.toLowerCase(); - - // If there's no demuxer/decoder node, any format might work (e.g., passthrough pipelines) - const hasDecoder = lowerYaml.includes('demux') || lowerYaml.includes('decode'); - if (!hasDecoder) { - return null; // Accept all formats - } - - const compatibleFormats: string[] = []; - - // OGG container (opus, vorbis) - // Match patterns: ogg::demuxer, ogg_demux, opus::decoder, opus_decode - if ( - lowerYaml.includes('ogg::demux') || - lowerYaml.includes('ogg_demux') || - lowerYaml.includes('opus::decode') || - lowerYaml.includes('opus_decode') - ) { - compatibleFormats.push('ogg', 'opus'); - } - - // FLAC - if (lowerYaml.includes('flac')) { - compatibleFormats.push('flac'); - } - - // WAV/PCM - if (lowerYaml.includes('wav') || lowerYaml.includes('pcm')) { - compatibleFormats.push('wav'); - } - - // MP3 - if (lowerYaml.includes('mp3')) { - compatibleFormats.push('mp3'); - } - - // If we found specific formats, return them; otherwise return null (accept all) - return compatibleFormats.length > 0 ? compatibleFormats : null; - }; - - /** - * Detects optional asset tags for Convert view's asset picker. - * - * This is a UI-only hint, carried in YAML comments so it doesn't affect pipeline parsing. - * - * Format: - * # skit:input_asset_tags=speech,music - */ - const detectInputAssetTags = (yaml: string): string[] | null => { - const match = yaml.match(/^\s*#\s*skit:input_asset_tags\s*=\s*([^\n#]+)\s*$/im); - if (!match?.[1]) return null; - - const tags = match[1] - .split(',') - .map((tag) => tag.trim().toLowerCase()) - .filter(Boolean); - - return tags.length > 0 ? tags : null; - }; - - const assetMatchesTag = (assetId: string, tag: string): boolean => { - if (tag === 'speech') { - return assetId.toLowerCase().startsWith('speech_'); - } - - if (tag === 'music') { - return assetId.toLowerCase().startsWith('music_'); - } - - if (tag.startsWith('id:')) { - return assetId.toLowerCase() === tag.slice('id:'.length).trim().toLowerCase(); - } - - return false; - }; - - // Filter assets based on pipeline's expected format; for multi-field uploads, allow all assets so fields can mix - const inputFormatSpec = React.useMemo(() => detectInputFormatSpec(pipelineYaml), [pipelineYaml]); - const inferredFormats = React.useMemo(() => detectExpectedFormats(pipelineYaml), [pipelineYaml]); - const assetFieldName = httpInputFields.length > 0 ? httpInputFields[0].name : 'media'; - const assetFormats = React.useMemo( - () => resolveFormatsForField(assetFieldName, inputFormatSpec, inferredFormats), - [assetFieldName, inputFormatSpec, inferredFormats] + // Derive a field-hint lookup from the client section + const getFieldHint = useCallback( + (fieldName: string): { accept?: string; hint?: string } => { + const hints = client?.input?.field_hints; + if (hints) { + const fh = hints[fieldName]; + if (fh) { + const formats = parseAcceptToFormats(fh.accept); + if (formats && formats.length > 0) { + const accept = formats.map((f) => `.${f}`).join(','); + const label = formats.map((f) => f.toUpperCase()).join(', '); + return { accept, hint: `Expected format${formats.length > 1 ? 's' : ''}: ${label}` }; + } + if (fh.accept) return { accept: fh.accept }; + } + } + // Fall back to top-level input accept + const topFormats = parseAcceptToFormats(client?.input?.accept); + if (topFormats && topFormats.length > 0) { + const accept = topFormats.map((f) => `.${f}`).join(','); + const label = topFormats.map((f) => f.toUpperCase()).join(', '); + return { accept, hint: `Expected format${topFormats.length > 1 ? 's' : ''}: ${label}` }; + } + return {}; + }, + [client] ); - const filteredAssets = React.useMemo(() => { - if (!pipelineYaml) { - return audioAssets; - } - const inputAssetTags = detectInputAssetTags(pipelineYaml); + // Filter assets based on client-declared accept & asset_tags + const assetFormats = useMemo(() => parseAcceptToFormats(client?.input?.accept), [client]); + const inputAssetTags = client?.input?.asset_tags ?? null; + const filteredAssets = useMemo(() => { + if (!pipelineYaml) return audioAssets; // Multi-field pipelines: only filter by format (avoid tag-based narrowing so users can mix content) if (httpInputFields.length > 1) { @@ -933,7 +701,6 @@ const ConvertView: React.FC = () => { viewsLogger.debug('Expected formats:', assetFormats, 'Total assets:', audioAssets.length); - // Filter assets to only those with compatible formats const formatFiltered = assetFormats ? audioAssets.filter((asset) => assetFormats.includes(asset.format.toLowerCase())) : audioAssets; @@ -947,7 +714,7 @@ const ConvertView: React.FC = () => { viewsLogger.debug('Filtered to', tagFiltered.length, 'compatible assets'); return tagFiltered; - }, [audioAssets, pipelineYaml, httpInputFields.length, assetFormats]); + }, [audioAssets, pipelineYaml, httpInputFields.length, assetFormats, inputAssetTags]); // Clear selected asset if it's no longer in the filtered list useEffect(() => { @@ -959,7 +726,7 @@ const ConvertView: React.FC = () => { // Track http_input fields for multi-upload pipelines useEffect(() => { - const { fields, hasHttpInput: hasHttp } = deriveHttpInputFields(pipelineYaml); + const { fields, hasHttpInput: hasHttp } = deriveHttpInputFieldsFromParsed(parsedPipelineYaml); setHasHttpInput(hasHttp); setHttpInputFields(fields); setFieldUploads((prev) => { @@ -969,32 +736,14 @@ const ConvertView: React.FC = () => { }); return next; }); - }, [pipelineYaml]); + }, [parsedPipelineYaml]); - // Watch for pipeline YAML changes and update transcription/TTS detection + // Force playback mode for transcription/TTS pipelines useEffect(() => { - const isTranscription = checkIfTranscriptionPipeline(pipelineYaml); - const isTTS = checkIfTTSPipeline(pipelineYaml); - const isNoInput = checkIfNoInputPipeline(pipelineYaml); - setIsTranscriptionPipeline(isTranscription); - setIsTTSPipeline(isTTS); - setIsNoInputPipeline(isNoInput); - // Force playback mode for transcription pipelines - if (isTranscription && outputMode !== 'playback') { + if ((isTranscriptionPipeline || isTTSPipeline) && outputMode !== 'playback') { setOutputMode('playback'); } - // TTS pipelines always output audio, so default to playback - if (isTTS && outputMode !== 'playback') { - setOutputMode('playback'); - } - }, [ - pipelineYaml, - outputMode, - setIsTranscriptionPipeline, - setIsTTSPipeline, - setIsNoInputPipeline, - setOutputMode, - ]); + }, [isTranscriptionPipeline, isTTSPipeline, outputMode, setOutputMode]); // Update YAML when asset selection changes useEffect(() => { @@ -1038,7 +787,7 @@ const ConvertView: React.FC = () => { const defaultSample = orderedSamples[0]; setSelectedTemplateId(defaultSample.id); setPipelineYaml(defaultSample.yaml); - setIsTranscriptionPipeline(checkIfTranscriptionPipeline(defaultSample.yaml)); + // Client-section flags are derived via useMemo, no manual setter needed } } catch (error) { viewsLogger.error('Failed to fetch samples:', error); @@ -1049,14 +798,7 @@ const ConvertView: React.FC = () => { }; fetchSamples(); - }, [ - setSamples, - setSamplesLoading, - setSamplesError, - setSelectedTemplateId, - setPipelineYaml, - setIsTranscriptionPipeline, - ]); + }, [setSamples, setSamplesLoading, setSamplesError, setSelectedTemplateId, setPipelineYaml]); const handleTemplateSelect = (templateId: string) => { const sample = samples.find((s) => s.id === templateId); @@ -1068,9 +810,10 @@ const ConvertView: React.FC = () => { // Set original YAML (asset selection will be reapplied via useEffect if needed) setPipelineYaml(sample.yaml); - setIsTranscriptionPipeline(checkIfTranscriptionPipeline(sample.yaml)); - // Force playback mode for transcription pipelines - if (checkIfTranscriptionPipeline(sample.yaml)) { + // Force playback for transcription pipelines (will be picked up by the + // effect once pipelineYaml updates and client is re-derived). + const templateClient = parseClientFromYaml(sample.yaml); + if (templateClient?.output?.type === 'transcription') { setOutputMode('playback'); } } @@ -1079,7 +822,8 @@ const ConvertView: React.FC = () => { const prepareUploads = useCallback(async (): Promise => { // For no-input (generator) pipelines, use the raw httpInputFields // so that truly input-less pipelines (no http_input node) send no uploads. - if (isNoInputPipeline) { + // Also handle custom pipelines without a client section that lack http_input. + if (isNoInputPipeline || (!client && !hasHttpInput)) { return buildNoInputUploads(httpInputFields); } @@ -1098,6 +842,7 @@ const ConvertView: React.FC = () => { } return []; }, [ + client, fieldUploads, httpInputFields, inputMode, @@ -1426,10 +1171,7 @@ const ConvertView: React.FC = () => { const ttsMissingRequiredUploads = ttsExtraFields.some( (field) => field.required && !fieldUploads[field.name] ); - const singleUploadHint = - uploadFields.length > 0 - ? formatHintForField(uploadFields[0], inputFormatSpec, inferredFormats, isTTSPipeline) - : {}; + const singleUploadHint = uploadFields.length > 0 ? getFieldHint(uploadFields[0].name) : {}; const handleDownloadAudio = () => { if (!mediaUrl) return; @@ -1604,12 +1346,7 @@ const ConvertView: React.FC = () => {
Additional uploads: {ttsExtraFields.map((field) => { - const hint = formatHintForField( - field, - inputFormatSpec, - inferredFormats, - isTTSPipeline - ); + const hint = getFieldHint(field.name); return (
@@ -1658,12 +1395,7 @@ const ConvertView: React.FC = () => { or pick an existing asset.

{uploadFields.map((field) => { - const hint = formatHintForField( - field, - inputFormatSpec, - inferredFormats, - isTTSPipeline - ); + const hint = getFieldHint(field.name); return (
diff --git a/ui/src/views/StreamView.tsx b/ui/src/views/StreamView.tsx index dc10c804..338ce704 100644 --- a/ui/src/views/StreamView.tsx +++ b/ui/src/views/StreamView.tsx @@ -31,13 +31,36 @@ import { createSession } from '@/services/sessions'; import { useSchemaStore, ensureSchemasLoaded } from '@/stores/schemaStore'; import type { Event } from '@/types/types'; import { getLogger } from '@/utils/logger'; -import { extractMoqPeerSettings, updateUrlPath } from '@/utils/moqPeerSettings'; +import { + extractMoqPeerSettings, + updateUrlPath, + type MoqPeerSettings, +} from '@/utils/moqPeerSettings'; import { orderSamplePipelinesSystemFirst } from '@/utils/samplePipelineOrdering'; import { useStreamStore } from '../stores/streamStore'; const logger = getLogger('StreamView'); +/** + * Resolves the server URL for a pipeline's MoQ settings. + * + * - Relay pipelines use the relay URL directly. + * - Gateway pipelines apply the gateway path to the original config URL + * (not the current serverUrl, which may have been overwritten by a + * previous relay selection). + * + * Returns the resolved URL or undefined if no update is needed. + */ +function resolveServerUrl(settings: MoqPeerSettings): string | undefined { + if (settings.relayUrl) return settings.relayUrl; + if (settings.gatewayPath) { + const baseUrl = useStreamStore.getState().configServerUrl; + if (baseUrl) return updateUrlPath(baseUrl, settings.gatewayPath); + } + return undefined; +} + const ConnectionControlsRow = styled.div` display: flex; align-items: center; @@ -269,6 +292,7 @@ const StreamView: React.FC = () => { watchStatus, pipelineNeedsAudio, pipelineNeedsVideo, + connectingStep, errorMessage, configLoaded, activeSessionId, @@ -309,6 +333,7 @@ const StreamView: React.FC = () => { watchStatus: s.watchStatus, pipelineNeedsAudio: s.pipelineNeedsAudio, pipelineNeedsVideo: s.pipelineNeedsVideo, + connectingStep: s.connectingStep, errorMessage: s.errorMessage, configLoaded: s.configLoaded, activeSessionId: s.activeSessionId, @@ -425,15 +450,8 @@ const StreamView: React.FC = () => { const moqSettings = extractMoqPeerSettings(first.yaml); if (moqSettings) { - if (moqSettings.relayUrl) { - setServerUrl(moqSettings.relayUrl); - } else if (moqSettings.gatewayPath && useStreamStore.getState().serverUrl) { - // Read serverUrl directly from the store since this effect - // runs on mount and the closure-captured value is still ''. - setServerUrl( - updateUrlPath(useStreamStore.getState().serverUrl, moqSettings.gatewayPath) - ); - } + const resolvedUrl = resolveServerUrl(moqSettings); + if (resolvedUrl) setServerUrl(resolvedUrl); if (moqSettings.inputBroadcast) { setInputBroadcast(moqSettings.inputBroadcast); } @@ -471,13 +489,8 @@ const StreamView: React.FC = () => { // Auto-adjust connection settings based on moq_peer node in the pipeline const moqSettings = extractMoqPeerSettings(template.yaml); if (moqSettings) { - // Update server URL: direct relay URL takes priority, otherwise - // apply the gateway path to the current server URL. - if (moqSettings.relayUrl) { - setServerUrl(moqSettings.relayUrl); - } else if (moqSettings.gatewayPath && serverUrl) { - setServerUrl(updateUrlPath(serverUrl, moqSettings.gatewayPath)); - } + const resolvedUrl = resolveServerUrl(moqSettings); + if (resolvedUrl) setServerUrl(resolvedUrl); // Update broadcast names if specified if (moqSettings.inputBroadcast) { setInputBroadcast(moqSettings.inputBroadcast); @@ -506,7 +519,6 @@ const StreamView: React.FC = () => { }, [ viewState, - serverUrl, setServerUrl, setInputBroadcast, setOutputBroadcast, @@ -548,9 +560,15 @@ const StreamView: React.FC = () => { if (status === 'disconnected' && serverUrl.trim()) { void (async () => { try { - await connect(); + const ok = await connect(); + if (!ok) { + logger.warn('Auto-connect after session creation did not succeed'); + } } catch (error) { logger.error('MoQ connection attempt after session creation failed:', error); + viewState.setSessionCreationError( + error instanceof Error ? error.message : 'Connection failed after session creation' + ); } })(); } @@ -638,6 +656,12 @@ const StreamView: React.FC = () => { live: 'Watch: live', }; + const connectingStepText: Record = { + devices: 'Requesting device access', + relay: 'Connecting to relay', + pipeline: 'Waiting for pipeline', + }; + return ( { {(status === 'connecting' || status === 'connected') && (
- {status === 'connected' ? 'Relay: connected' : 'Relay: connecting…'} •{' '} - {watchStatusText[watchStatus]} + {status === 'connected' + ? 'Relay: connected' + : connectingStep + ? 'Connecting — ' + (connectingStepText[connectingStep] ?? connectingStep) + : 'Connecting…'}{' '} + • {watchStatusText[watchStatus]} {pipelineNeedsAudio && <> • {micStatusText[micStatus]}} {pipelineNeedsVideo && <> • {cameraStatusText[cameraStatus]}}