From 68efa9b4a1c131774f3837584453159f8ac3f85b Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:12:04 +0000 Subject: [PATCH 01/20] feat: add declarative client section schema (Phase 1) - Add ClientSection, PublishConfig, WatchConfig, InputConfig, OutputConfig, FieldHint, FieldType types to yaml.rs with serde + ts-rs derives - Add client field to UserPipeline::Steps and UserPipeline::Dag variants - Add client field to compiled Pipeline struct in lib.rs - Update compile(), compile_steps(), compile_dag() to forward client - Add pre-validation in parse_yaml() for better error messages - Update populate_session_pipeline() to preserve name/description/mode/client - Update TS type generator to export all client section types - Add clientSection.ts helpers (extractClientSection, deriveSettingsFromClient) - Add comprehensive Rust + TS unit tests Co-Authored-By: Claudio Costa --- apps/skit/src/server.rs | 6 + crates/api/src/bin/generate_ts_types.rs | 10 + crates/api/src/lib.rs | 4 + crates/api/src/yaml.rs | 340 +++++++++++++++++++++++- ui/src/types/generated/api-types.ts | 113 +++++++- ui/src/utils/clientSection.test.ts | 145 ++++++++++ ui/src/utils/clientSection.ts | 36 +++ 7 files changed, 645 insertions(+), 9 deletions(-) create mode 100644 ui/src/utils/clientSection.test.ts create mode 100644 ui/src/utils/clientSection.ts diff --git a/apps/skit/src/server.rs b/apps/skit/src/server.rs index ba151091..68228cf5 100644 --- a/apps/skit/src/server.rs +++ b/apps/skit/src/server.rs @@ -1520,6 +1520,12 @@ struct CreateSessionResponse { async fn populate_session_pipeline(session: &crate::session::Session, engine_pipeline: &Pipeline) { let mut pipeline = session.pipeline.lock().await; + // Forward top-level metadata so the UI can read it from the session snapshot. + pipeline.name = engine_pipeline.name.clone(); + pipeline.description = engine_pipeline.description.clone(); + pipeline.mode = engine_pipeline.mode; + pipeline.client = engine_pipeline.client.clone(); + // Add nodes to in-memory pipeline for (node_id, node_spec) in &engine_pipeline.nodes { pipeline.nodes.insert( diff --git a/crates/api/src/bin/generate_ts_types.rs b/crates/api/src/bin/generate_ts_types.rs index 02943730..7354fdbe 100644 --- a/crates/api/src/bin/generate_ts_types.rs +++ b/crates/api/src/bin/generate_ts_types.rs @@ -63,6 +63,16 @@ fn main() -> Result<(), Box> { format!("export {}", streamkit_api::ValidationError::decl(&cfg)), format!("export {}", streamkit_api::ValidationErrorType::decl(&cfg)), format!("export {}", streamkit_api::PermissionsInfo::decl(&cfg)), + // Declarative client section types + format!("\n// client section\nexport {}", streamkit_api::yaml::ClientSection::decl(&cfg)), + format!("export {}", streamkit_api::yaml::PublishConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::WatchConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::InputConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::InputType::decl(&cfg)), + format!("export {}", streamkit_api::yaml::OutputConfig::decl(&cfg)), + format!("export {}", streamkit_api::yaml::OutputType::decl(&cfg)), + format!("export {}", streamkit_api::yaml::FieldHint::decl(&cfg)), + format!("export {}", streamkit_api::yaml::FieldType::decl(&cfg)), ]; let output = declarations.join("\n\n"); diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 07af1903..b72706ce 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -538,6 +538,10 @@ pub struct Pipeline { pub description: Option, #[serde(default)] pub mode: EngineMode, + /// Declarative UI metadata — forwarded unchanged from `UserPipeline`, + /// ignored by the engine for execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub client: Option, #[ts(type = "Record")] pub nodes: indexmap::IndexMap, pub connections: Vec, diff --git a/crates/api/src/yaml.rs b/crates/api/src/yaml.rs index 86500178..04ced9b6 100644 --- a/crates/api/src/yaml.rs +++ b/crates/api/src/yaml.rs @@ -11,7 +11,8 @@ use super::{Connection, ConnectionMode, EngineMode, Node, Pipeline}; use indexmap::IndexMap; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; /// Represents a single step in a linear pipeline definition. #[derive(Debug, Deserialize)] @@ -92,6 +93,146 @@ pub enum Needs { Map(IndexMap), } +// --------------------------------------------------------------------------- +// Declarative `client` section — UI metadata for pipeline rendering +// --------------------------------------------------------------------------- + +/// Top-level `client` section in pipeline YAML. +/// +/// Declares what the browser UI should do when rendering this pipeline. +/// Dynamic pipelines use `relay_url`/`gateway_path`/`publish`/`watch`; +/// oneshot pipelines use `input`/`output`. The two sets are mutually +/// exclusive by mode (enforced by the lint pass, not at parse time). +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct ClientSection { + /// Direct relay URL for external MoQ relay pattern. + pub relay_url: Option, + /// Gateway path for gateway-managed MoQ pattern. + pub gateway_path: Option, + /// Browser-side publish configuration (dynamic pipelines). + pub publish: Option, + /// Browser-side watch configuration (dynamic pipelines). + pub watch: Option, + /// Input UX configuration (oneshot pipelines). + pub input: Option, + /// Output rendering configuration (oneshot pipelines). + pub output: Option, +} + +/// Browser-side publish configuration for dynamic pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct PublishConfig { + /// Broadcast name the browser publishes to. + pub broadcast: String, + /// Whether the pipeline consumes audio from the browser. + #[serde(default)] + pub audio: bool, + /// Whether the pipeline consumes video from the browser. + #[serde(default)] + pub video: bool, +} + +/// Browser-side watch configuration for dynamic pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct WatchConfig { + /// Broadcast name the browser subscribes to. + pub broadcast: String, + /// Whether the pipeline outputs audio to subscribers. + #[serde(default)] + pub audio: bool, + /// Whether the pipeline outputs video to subscribers. + #[serde(default)] + pub video: bool, +} + +/// Input UX configuration for oneshot pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct InputConfig { + /// The kind of input UX to present. + #[serde(rename = "type")] + pub input_type: InputType, + /// MIME filter for file pickers (e.g. `audio/*`). + pub accept: Option, + /// Tags for filtering the asset picker (e.g. `["speech"]`). + pub asset_tags: Option>, + /// Placeholder text for text inputs. + pub placeholder: Option, + /// Per-field UI hints keyed by `http_input` field name. + #[ts(type = "Record | null")] + pub field_hints: Option>, +} + +/// The kind of input UX a oneshot pipeline expects. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS)] +#[ts(export)] +#[serde(rename_all = "snake_case")] +pub enum InputType { + /// File upload with optional MIME filter. + FileUpload, + /// Free-form text input. + Text, + /// Has `http_input` but the body is irrelevant (trigger only). + Trigger, + /// No `http_input` node — pipeline generates its own input. + None, +} + +/// Output rendering configuration for oneshot pipelines. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct OutputConfig { + /// The media kind the pipeline produces. + #[serde(rename = "type")] + pub output_type: OutputType, +} + +/// The media kind a oneshot pipeline produces. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS)] +#[ts(export)] +#[serde(rename_all = "snake_case")] +pub enum OutputType { + /// Transcription segments (JSON stream). + Transcription, + /// Generic JSON stream. + Json, + /// Audio output. + Audio, + /// Video output. + Video, +} + +/// Per-field input type discriminator for `field_hints`. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, TS)] +#[ts(export)] +#[serde(rename_all = "snake_case")] +pub enum FieldType { + /// File upload field. + File, + /// Text input field. + Text, +} + +/// Per-field UI hint within `InputConfig.field_hints`. +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct FieldHint { + /// Override the field's input type (default is file upload). + #[serde(rename = "type")] + pub field_type: Option, + /// MIME filter for file picker. + pub accept: Option, + /// Placeholder text for text inputs. + pub placeholder: Option, +} + +// --------------------------------------------------------------------------- +// User-facing pipeline definition +// --------------------------------------------------------------------------- + /// The top-level structure for a user-facing pipeline definition. /// `serde(untagged)` allows it to be parsed as either a steps-based /// pipeline or a nodes-based (DAG) pipeline. @@ -106,6 +247,8 @@ pub enum UserPipeline { #[serde(default)] mode: EngineMode, steps: Vec, + /// Declarative UI metadata (optional — required only for UI rendering). + client: Option, }, Dag { #[serde(skip_serializing_if = "Option::is_none")] @@ -115,6 +258,8 @@ pub enum UserPipeline { #[serde(default)] mode: EngineMode, nodes: IndexMap, + /// Declarative UI metadata (optional — required only for UI rendering). + client: Option, }, } @@ -131,6 +276,15 @@ pub enum UserPipeline { pub fn parse_yaml(yaml: &str) -> Result { let json_value: serde_json::Value = serde_saphyr::from_str(yaml).map_err(|e| format!("Invalid YAML: {e}"))?; + + // Pre-validate `client` if present so that enum/type errors produce + // actionable messages instead of collapsing into the generic + // "did not match any variant" error from the untagged `UserPipeline`. + if let Some(client_val) = json_value.get("client") { + let _: ClientSection = serde_json::from_value(client_val.clone()) + .map_err(|e| format!("Invalid client section: {e}"))?; + } + serde_json::from_value(json_value).map_err(|e| format!("Invalid pipeline: {e}")) } @@ -141,11 +295,11 @@ pub fn parse_yaml(yaml: &str) -> Result { /// Returns an error if a node references a non-existent dependency in its `needs` field. pub fn compile(pipeline: UserPipeline) -> Result { match pipeline { - UserPipeline::Steps { name, description, mode, steps } => { - Ok(compile_steps(name, description, mode, steps)) + UserPipeline::Steps { name, description, mode, steps, client } => { + Ok(compile_steps(name, description, mode, steps, client)) }, - UserPipeline::Dag { name, description, mode, nodes } => { - compile_dag(name, description, mode, nodes) + UserPipeline::Dag { name, description, mode, nodes, client } => { + compile_dag(name, description, mode, nodes, client) }, } } @@ -156,6 +310,7 @@ fn compile_steps( description: Option, mode: EngineMode, steps: Vec, + client: Option, ) -> Pipeline { let mut nodes = IndexMap::new(); let mut connections = Vec::new(); @@ -177,7 +332,7 @@ fn compile_steps( nodes.insert(node_name, Node { kind: step.kind, params: step.params, state: None }); } - Pipeline { name, description, mode, nodes, connections, view_data: None } + Pipeline { name, description, mode, client, nodes, connections, view_data: None } } /// Known bidirectional node kinds that are allowed to participate in cycles. @@ -300,6 +455,7 @@ fn compile_dag( description: Option, mode: EngineMode, user_nodes: IndexMap, + client: Option, ) -> Result { // First, detect cycles in the dependency graph detect_cycles(&user_nodes)?; @@ -409,7 +565,7 @@ fn compile_dag( }) .collect(); - Ok(Pipeline { name, description, mode, nodes, connections, view_data: None }) + Ok(Pipeline { name, description, mode, client, nodes, connections, view_data: None }) } #[cfg(test)] @@ -968,4 +1124,174 @@ nodes: assert_eq!(conn.to_node, "sink"); assert_eq!(conn.to_pin, "my_input"); } + + // ----------------------------------------------------------------------- + // Client section tests + // ----------------------------------------------------------------------- + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_parsed_in_steps() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input + - kind: streamkit::http_output +client: + input: + type: file_upload + accept: "audio/*" + output: + type: transcription +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + let input = client.input.expect("input config should be present"); + assert!(matches!(input.input_type, InputType::FileUpload)); + assert_eq!(input.accept.as_deref(), Some("audio/*")); + + let output = client.output.expect("output config should be present"); + assert!(matches!(output.output_type, OutputType::Transcription)); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_parsed_in_dag() { + let yaml = r#" +mode: dynamic +nodes: + peer: + kind: transport::moq::peer + params: + gateway_path: /moq/test + input_broadcast: camera + output_broadcast: output +client: + gateway_path: /moq/test + publish: + broadcast: camera + audio: true + video: true + watch: + broadcast: output + audio: true + video: true +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + assert_eq!(client.gateway_path.as_deref(), Some("/moq/test")); + + let publish = client.publish.expect("publish config should be present"); + assert_eq!(publish.broadcast, "camera"); + assert!(publish.audio); + assert!(publish.video); + + let watch = client.watch.expect("watch config should be present"); + assert_eq!(watch.broadcast, "output"); + assert!(watch.audio); + assert!(watch.video); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_optional() { + let yaml = r" +mode: oneshot +steps: + - kind: core::passthrough +"; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + assert!(compiled.client.is_none()); + } + + #[test] + fn test_invalid_client_section_rejected() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input +client: + input: + type: invalid_type +"#; + let result = parse_yaml(yaml); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.contains("Invalid client section"), + "Error should mention client section: {err}" + ); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_with_field_hints() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input + - kind: streamkit::http_output +client: + input: + type: file_upload + accept: "audio/*" + field_hints: + text: + type: text + placeholder: "Enter your prompt" + reference: + type: file + accept: "audio/*" + output: + type: audio +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + let input = client.input.expect("input config should be present"); + let hints = input.field_hints.expect("field_hints should be present"); + + assert_eq!(hints.len(), 2); + + let text_hint = hints.get("text").expect("text hint should exist"); + assert!(matches!(text_hint.field_type, Some(FieldType::Text))); + assert_eq!(text_hint.placeholder.as_deref(), Some("Enter your prompt")); + + let ref_hint = hints.get("reference").expect("reference hint should exist"); + assert!(matches!(ref_hint.field_type, Some(FieldType::File))); + assert_eq!(ref_hint.accept.as_deref(), Some("audio/*")); + } + + #[test] + #[allow(clippy::unwrap_used)] + fn test_client_section_with_asset_tags() { + let yaml = r#" +mode: oneshot +steps: + - kind: streamkit::http_input + - kind: streamkit::http_output +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + - voice + output: + type: transcription +"#; + let pipeline = parse_yaml(yaml).unwrap(); + let compiled = compile(pipeline).unwrap(); + + let client = compiled.client.expect("client section should be present"); + let input = client.input.expect("input config should be present"); + let tags = input.asset_tags.expect("asset_tags should be present"); + assert_eq!(tags, vec!["speech", "voice"]); + } } diff --git a/ui/src/types/generated/api-types.ts b/ui/src/types/generated/api-types.ts index edac33f0..c85c5203 100644 --- a/ui/src/types/generated/api-types.ts +++ b/ui/src/types/generated/api-types.ts @@ -363,7 +363,12 @@ export type Node = { kind: string, params: JsonValue, */ state: NodeState | null, }; -export type Pipeline = { name: string | null, description: string | null, mode: EngineMode, nodes: Record, connections: Array, +export type Pipeline = { name: string | null, description: string | null, mode: EngineMode, +/** + * Declarative UI metadata — forwarded unchanged from `UserPipeline`, + * ignored by the engine for execution. + */ +client: ClientSection | null, nodes: Record, connections: Array, /** * Resolved per-node view data (e.g., compositor layout). * Only populated in API responses; absent from pipeline definitions. @@ -418,4 +423,108 @@ export type ValidationError = { error_type: ValidationErrorType, message: string export type ValidationErrorType = "error" | "warning"; -export type PermissionsInfo = { create_sessions: boolean, destroy_sessions: boolean, list_sessions: boolean, modify_sessions: boolean, tune_nodes: boolean, load_plugins: boolean, delete_plugins: boolean, list_nodes: boolean, list_samples: boolean, read_samples: boolean, write_samples: boolean, delete_samples: boolean, access_all_sessions: boolean, upload_assets: boolean, delete_assets: boolean, }; \ No newline at end of file +export type PermissionsInfo = { create_sessions: boolean, destroy_sessions: boolean, list_sessions: boolean, modify_sessions: boolean, tune_nodes: boolean, load_plugins: boolean, delete_plugins: boolean, list_nodes: boolean, list_samples: boolean, read_samples: boolean, write_samples: boolean, delete_samples: boolean, access_all_sessions: boolean, upload_assets: boolean, delete_assets: boolean, }; + + +// client section +export type ClientSection = { +/** + * Direct relay URL for external MoQ relay pattern. + */ +relay_url: string | null, +/** + * Gateway path for gateway-managed MoQ pattern. + */ +gateway_path: string | null, +/** + * Browser-side publish configuration (dynamic pipelines). + */ +publish: PublishConfig | null, +/** + * Browser-side watch configuration (dynamic pipelines). + */ +watch: WatchConfig | null, +/** + * Input UX configuration (oneshot pipelines). + */ +input: InputConfig | null, +/** + * Output rendering configuration (oneshot pipelines). + */ +output: OutputConfig | null, }; + +export type PublishConfig = { +/** + * Broadcast name the browser publishes to. + */ +broadcast: string, +/** + * Whether the pipeline consumes audio from the browser. + */ +audio: boolean, +/** + * Whether the pipeline consumes video from the browser. + */ +video: boolean, }; + +export type WatchConfig = { +/** + * Broadcast name the browser subscribes to. + */ +broadcast: string, +/** + * Whether the pipeline outputs audio to subscribers. + */ +audio: boolean, +/** + * Whether the pipeline outputs video to subscribers. + */ +video: boolean, }; + +export type InputConfig = { +/** + * The kind of input UX to present. + */ +type: InputType, +/** + * MIME filter for file pickers (e.g. `audio/*`). + */ +accept: string | null, +/** + * Tags for filtering the asset picker (e.g. `["speech"]`). + */ +asset_tags: Array | null, +/** + * Placeholder text for text inputs. + */ +placeholder: string | null, +/** + * Per-field UI hints keyed by `http_input` field name. + */ +field_hints: Record | null, }; + +export type InputType = "file_upload" | "text" | "trigger" | "none"; + +export type OutputConfig = { +/** + * The media kind the pipeline produces. + */ +type: OutputType, }; + +export type OutputType = "transcription" | "json" | "audio" | "video"; + +export type FieldHint = { +/** + * Override the field's input type (default is file upload). + */ +type: FieldType | null, +/** + * MIME filter for file picker. + */ +accept: string | null, +/** + * Placeholder text for text inputs. + */ +placeholder: string | null, }; + +export type FieldType = "file" | "text"; \ No newline at end of file diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts new file mode 100644 index 00000000..f4c56b1c --- /dev/null +++ b/ui/src/utils/clientSection.test.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +import { describe, expect, it } from 'vitest'; + +import type { ClientSection } from '@/types/types'; + +import { deriveSettingsFromClient, extractClientSection } from './clientSection'; + +describe('extractClientSection', () => { + it('returns null for null pipeline', () => { + expect(extractClientSection(null)).toBeNull(); + }); + + it('returns null for undefined pipeline', () => { + expect(extractClientSection(undefined)).toBeNull(); + }); + + it('returns null when pipeline has no client', () => { + const pipeline = { nodes: {}, connections: [], mode: 'live' } as never; + expect(extractClientSection(pipeline)).toBeNull(); + }); + + it('returns client section when present', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/test', + publish: null, + watch: null, + input: null, + output: null, + }; + const pipeline = { nodes: {}, connections: [], mode: 'live', client } as never; + expect(extractClientSection(pipeline)).toBe(client); + }); +}); + +describe('deriveSettingsFromClient', () => { + it('derives settings for a gateway publish+watch pipeline', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/compositor', + publish: { broadcast: 'camera-feed', audio: true, video: true }, + watch: { broadcast: 'composited-output', audio: true, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: '/moq/compositor', + relayUrl: undefined, + inputBroadcast: 'camera-feed', + outputBroadcast: 'composited-output', + hasInputBroadcast: true, + needsAudioInput: true, + needsVideoInput: true, + outputsAudio: true, + outputsVideo: true, + }); + }); + + it('derives settings for a relay-based pipeline', () => { + const client: ClientSection = { + relay_url: 'https://relay.example.com', + gateway_path: null, + publish: { broadcast: 'input', audio: true, video: false }, + watch: { broadcast: 'output', audio: false, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: undefined, + relayUrl: 'https://relay.example.com', + inputBroadcast: 'input', + outputBroadcast: 'output', + hasInputBroadcast: true, + needsAudioInput: true, + needsVideoInput: false, + outputsAudio: false, + outputsVideo: true, + }); + }); + + it('derives settings for a watch-only pipeline', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/monitor', + publish: null, + watch: { broadcast: 'preview', audio: false, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: '/moq/monitor', + relayUrl: undefined, + inputBroadcast: undefined, + outputBroadcast: 'preview', + hasInputBroadcast: false, + needsAudioInput: false, + needsVideoInput: false, + outputsAudio: false, + outputsVideo: true, + }); + }); + + it('derives settings for a oneshot pipeline (no publish/watch)', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: null, + publish: null, + watch: null, + input: { + type: 'file_upload', + accept: 'audio/*', + asset_tags: null, + placeholder: null, + field_hints: null, + }, + output: { type: 'transcription' }, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings).toEqual({ + gatewayPath: undefined, + relayUrl: undefined, + inputBroadcast: undefined, + outputBroadcast: undefined, + hasInputBroadcast: false, + needsAudioInput: false, + needsVideoInput: false, + outputsAudio: false, + outputsVideo: false, + }); + }); +}); diff --git a/ui/src/utils/clientSection.ts b/ui/src/utils/clientSection.ts new file mode 100644 index 00000000..1c6b38ee --- /dev/null +++ b/ui/src/utils/clientSection.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: © 2025 StreamKit Contributors +// +// SPDX-License-Identifier: MPL-2.0 + +import type { ClientSection, Pipeline } from '@/types/types'; + +import type { MoqPeerSettings } from './moqPeerSettings'; + +/** + * Extracts the `client` section from a compiled `Pipeline`, returning `null` + * when the field is absent. + */ +export function extractClientSection(pipeline: Pipeline | null | undefined): ClientSection | null { + return pipeline?.client ?? null; +} + +/** + * Derives `MoqPeerSettings` from a declarative `ClientSection`. + * + * This is the client-section counterpart of `extractMoqPeerSettings()` (which + * heuristically inspects raw YAML). When a pipeline carries a `client` + * section the UI should prefer this function. + */ +export function deriveSettingsFromClient(client: ClientSection): MoqPeerSettings { + return { + gatewayPath: client.gateway_path ?? undefined, + relayUrl: client.relay_url ?? undefined, + inputBroadcast: client.publish?.broadcast, + outputBroadcast: client.watch?.broadcast, + hasInputBroadcast: Boolean(client.publish), + needsAudioInput: client.publish?.audio ?? false, + needsVideoInput: client.publish?.video ?? false, + outputsAudio: client.watch?.audio ?? false, + outputsVideo: client.watch?.video ?? false, + }; +} From 01b7e7bfb6437c7ad9ba742b1c0f3b77ba03c8e9 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:14:58 +0000 Subject: [PATCH 02/20] feat: add client sections to all 34 pipeline YAML files (Phase 2) - Add client sections to all 16 dynamic pipelines (gateway + relay patterns) - Add client sections to all 18 oneshot pipelines (file_upload, text, trigger, none) - Remove all 9 '# skit:' magic comments, replaced by client.input.asset_tags and client.input.field_hints - Classify each pipeline with appropriate input/output types Co-Authored-By: Claudio Costa --- samples/pipelines/dynamic/moq.yml | 10 ++++++++++ samples/pipelines/dynamic/moq_mixing.yml | 10 ++++++++++ samples/pipelines/dynamic/moq_peer.yml | 10 ++++++++++ samples/pipelines/dynamic/moq_relay_echo.yml | 10 ++++++++++ samples/pipelines/dynamic/moq_relay_webcam_pip.yml | 10 ++++++++++ .../pipelines/dynamic/speech-translate-en-es.yaml | 10 ++++++++++ .../pipelines/dynamic/speech-translate-es-en.yaml | 10 ++++++++++ .../dynamic/speech-translate-helsinki-en-es.yaml | 10 ++++++++++ .../dynamic/speech-translate-helsinki-es-en.yaml | 10 ++++++++++ samples/pipelines/dynamic/video_moq_colorbars.yml | 6 ++++++ samples/pipelines/dynamic/video_moq_compositor.yml | 6 ++++++ .../dynamic/video_moq_webcam_circle_pip.yml | 10 ++++++++++ samples/pipelines/dynamic/video_moq_webcam_pip.yml | 10 ++++++++++ samples/pipelines/dynamic/voice-agent-openai.yaml | 10 ++++++++++ .../dynamic/voice-weather-open-meteo.yaml | 10 ++++++++++ samples/pipelines/oneshot/double_volume.yml | 6 ++++++ samples/pipelines/oneshot/dual_upload_mixing.yml | 11 ++++++++--- samples/pipelines/oneshot/gain_filter_rust.yml | 6 ++++++ samples/pipelines/oneshot/kokoro-tts.yml | 6 ++++++ samples/pipelines/oneshot/matcha-tts.yml | 6 ++++++ samples/pipelines/oneshot/mixing.yml | 11 ++++++++--- samples/pipelines/oneshot/piper-tts.yml | 6 ++++++ .../pipelines/oneshot/pocket-tts-voice-clone.yml | 14 ++++++++++++-- samples/pipelines/oneshot/pocket-tts.yml | 6 ++++++ samples/pipelines/oneshot/sensevoice-stt.yml | 11 ++++++++--- samples/pipelines/oneshot/speech_to_text.yml | 11 ++++++++--- .../pipelines/oneshot/speech_to_text_translate.yml | 11 ++++++++--- .../oneshot/speech_to_text_translate_helsinki.yml | 11 ++++++++--- samples/pipelines/oneshot/supertonic-tts.yml | 6 ++++++ samples/pipelines/oneshot/useless-facts-tts.yml | 5 +++++ samples/pipelines/oneshot/vad-demo.yml | 11 ++++++++--- samples/pipelines/oneshot/vad-filtered-stt.yml | 11 ++++++++--- samples/pipelines/oneshot/video_colorbars.yml | 5 +++++ .../pipelines/oneshot/video_compositor_demo.yml | 5 +++++ 34 files changed, 275 insertions(+), 26 deletions(-) diff --git a/samples/pipelines/dynamic/moq.yml b/samples/pipelines/dynamic/moq.yml index 4a7f139f..484f1ac7 100644 --- a/samples/pipelines/dynamic/moq.yml +++ b/samples/pipelines/dynamic/moq.yml @@ -1,6 +1,16 @@ name: Real-Time MoQ Transcoder description: Decodes a MoQ input broadcast, applies gain, and republishes to MoQ mode: dynamic +client: + gateway_path: /moq/transcoder + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: moq_peer: kind: transport::moq::peer diff --git a/samples/pipelines/dynamic/moq_mixing.yml b/samples/pipelines/dynamic/moq_mixing.yml index 88fca62f..c14ee79f 100644 --- a/samples/pipelines/dynamic/moq_mixing.yml +++ b/samples/pipelines/dynamic/moq_mixing.yml @@ -1,6 +1,16 @@ name: Real-Time MoQ Mixing (Mic + Music) description: Mixes a live MoQ input stream with a local music track and republishes to MoQ mode: dynamic +client: + gateway_path: /moq/mixing + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ # INPUTS: MoQ mic input + local music file diff --git a/samples/pipelines/dynamic/moq_peer.yml b/samples/pipelines/dynamic/moq_peer.yml index caaafac4..8213d58f 100644 --- a/samples/pipelines/dynamic/moq_peer.yml +++ b/samples/pipelines/dynamic/moq_peer.yml @@ -5,6 +5,16 @@ name: MoQ Peer Transcoder (Gateway) description: Accepts a WebTransport client via transport::moq::peer, applies gain, and loops processed audio back to subscribers mode: dynamic +client: + gateway_path: /moq + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # Bidirectional node that accepts WebTransport connections and provides: diff --git a/samples/pipelines/dynamic/moq_relay_echo.yml b/samples/pipelines/dynamic/moq_relay_echo.yml index ac398037..21311187 100644 --- a/samples/pipelines/dynamic/moq_relay_echo.yml +++ b/samples/pipelines/dynamic/moq_relay_echo.yml @@ -5,6 +5,16 @@ name: MoQ Relay Echo (Audio) description: Subscribes to an audio broadcast on a MoQ relay, applies unity gain, and republishes — validates the full relay pub/sub path mode: dynamic +client: + relay_url: http://localhost:4443 + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: moq_input: kind: transport::moq::subscriber diff --git a/samples/pipelines/dynamic/moq_relay_webcam_pip.yml b/samples/pipelines/dynamic/moq_relay_webcam_pip.yml index 8a62084a..54038e4f 100644 --- a/samples/pipelines/dynamic/moq_relay_webcam_pip.yml +++ b/samples/pipelines/dynamic/moq_relay_webcam_pip.yml @@ -7,6 +7,16 @@ description: Subscribes audio+video from a MoQ relay, composites the webcam as P # NOTE: The publisher must already be streaming to the 'input' broadcast before # starting this pipeline, otherwise track discovery will miss the video/data pin. mode: dynamic +client: + relay_url: http://localhost:4443 + publish: + broadcast: input + audio: true + video: true + watch: + broadcast: output + audio: true + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/speech-translate-en-es.yaml b/samples/pipelines/dynamic/speech-translate-en-es.yaml index b5bcf74b..603f6f16 100644 --- a/samples/pipelines/dynamic/speech-translate-en-es.yaml +++ b/samples/pipelines/dynamic/speech-translate-en-es.yaml @@ -23,6 +23,16 @@ name: Real-Time Speech Translation (English → Spanish) description: Real-time speech translation from English to Spanish via MoQ mode: dynamic +client: + gateway_path: /moq/speech-translate-en-es + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/speech-translate-es-en.yaml b/samples/pipelines/dynamic/speech-translate-es-en.yaml index 4e5cd217..27b7228f 100644 --- a/samples/pipelines/dynamic/speech-translate-es-en.yaml +++ b/samples/pipelines/dynamic/speech-translate-es-en.yaml @@ -24,6 +24,16 @@ name: Real-Time Speech Translation (Spanish → English) description: Real-time speech translation from Spanish to English via MoQ mode: dynamic +client: + gateway_path: /moq/speech-translate-es-en + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml b/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml index 111a73bb..b1056757 100644 --- a/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml +++ b/samples/pipelines/dynamic/speech-translate-helsinki-en-es.yaml @@ -27,6 +27,16 @@ name: Real-Time Speech Translation (English → Spanish) - Helsinki description: Real-time speech translation from English to Spanish using Apache 2.0 licensed Helsinki OPUS-MT mode: dynamic +client: + gateway_path: /moq/speech-translate-helsinki-en-es + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml b/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml index 61d959ed..bed2ea7a 100644 --- a/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml +++ b/samples/pipelines/dynamic/speech-translate-helsinki-es-en.yaml @@ -27,6 +27,16 @@ name: Real-Time Speech Translation (Spanish → English) - Helsinki description: Real-time speech translation from Spanish to English using Apache 2.0 licensed Helsinki OPUS-MT mode: dynamic +client: + gateway_path: /moq/speech-translate-helsinki-es-en + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ diff --git a/samples/pipelines/dynamic/video_moq_colorbars.yml b/samples/pipelines/dynamic/video_moq_colorbars.yml index 1a4921ef..eb18cf84 100644 --- a/samples/pipelines/dynamic/video_moq_colorbars.yml +++ b/samples/pipelines/dynamic/video_moq_colorbars.yml @@ -5,6 +5,12 @@ name: Video Color Bars (MoQ Stream) description: Continuously generates SMPTE color bars and streams via MoQ mode: dynamic +client: + gateway_path: /moq/video + watch: + broadcast: output + audio: false + video: true nodes: colorbars: diff --git a/samples/pipelines/dynamic/video_moq_compositor.yml b/samples/pipelines/dynamic/video_moq_compositor.yml index 22ca5c1b..aa5d96c6 100644 --- a/samples/pipelines/dynamic/video_moq_compositor.yml +++ b/samples/pipelines/dynamic/video_moq_compositor.yml @@ -5,6 +5,12 @@ name: Video Compositor (MoQ Stream) description: Composites two colorbars sources through the compositor node and streams via MoQ mode: dynamic +client: + gateway_path: /moq/video + watch: + broadcast: output + audio: false + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml b/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml index 4c4d6c6d..37fc9f09 100644 --- a/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml +++ b/samples/pipelines/dynamic/video_moq_webcam_circle_pip.yml @@ -5,6 +5,16 @@ name: Webcam Circle PiP (MoQ Stream) description: Composites the user's webcam as a circular picture-in-picture overlay (Loom-style) over colorbars, loops audio through a gain filter, and streams both via MoQ mode: dynamic +client: + gateway_path: /moq/video + publish: + broadcast: input + audio: true + video: true + watch: + broadcast: output + audio: true + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/video_moq_webcam_pip.yml b/samples/pipelines/dynamic/video_moq_webcam_pip.yml index b342adbf..184d8642 100644 --- a/samples/pipelines/dynamic/video_moq_webcam_pip.yml +++ b/samples/pipelines/dynamic/video_moq_webcam_pip.yml @@ -5,6 +5,16 @@ name: Webcam PiP (MoQ Stream) description: Composites the user's webcam as picture-in-picture over colorbars with a text overlay, loops audio through a gain filter, and streams both via MoQ mode: dynamic +client: + gateway_path: /moq/video + publish: + broadcast: input + audio: true + video: true + watch: + broadcast: output + audio: true + video: true nodes: colorbars_bg: diff --git a/samples/pipelines/dynamic/voice-agent-openai.yaml b/samples/pipelines/dynamic/voice-agent-openai.yaml index 4a73d601..63ba61c5 100644 --- a/samples/pipelines/dynamic/voice-agent-openai.yaml +++ b/samples/pipelines/dynamic/voice-agent-openai.yaml @@ -12,6 +12,16 @@ name: Real-Time Voice Agent (OpenAI) description: Runs a MoQ voice agent using Whisper STT, OpenAI, and Kokoro TTS mode: dynamic +client: + gateway_path: /moq/voice-agent-openai + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ # INPUT: Receive audio from MoQ broadcast diff --git a/samples/pipelines/dynamic/voice-weather-open-meteo.yaml b/samples/pipelines/dynamic/voice-weather-open-meteo.yaml index f5590e89..d4e6ce13 100644 --- a/samples/pipelines/dynamic/voice-weather-open-meteo.yaml +++ b/samples/pipelines/dynamic/voice-weather-open-meteo.yaml @@ -30,6 +30,16 @@ name: Real-Time Voice Weather (Open-Meteo) description: Ask for the weather in a location (STT → Open-Meteo → TTS) via MoQ mode: dynamic +client: + gateway_path: /moq/voice-weather-open-meteo + publish: + broadcast: input + audio: true + video: false + watch: + broadcast: output + audio: true + video: false nodes: # ============================================================ # INPUT: Receive audio from MoQ broadcast diff --git a/samples/pipelines/oneshot/double_volume.yml b/samples/pipelines/oneshot/double_volume.yml index a5890c07..b2e4b1b3 100644 --- a/samples/pipelines/oneshot/double_volume.yml +++ b/samples/pipelines/oneshot/double_volume.yml @@ -1,6 +1,12 @@ name: Volume Boost (2×) description: Boosts the volume of an uploaded Ogg/Opus file and returns Ogg/Opus mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio steps: - kind: streamkit::http_input - kind: containers::ogg::demuxer diff --git a/samples/pipelines/oneshot/dual_upload_mixing.yml b/samples/pipelines/oneshot/dual_upload_mixing.yml index d02534be..062ef25f 100644 --- a/samples/pipelines/oneshot/dual_upload_mixing.yml +++ b/samples/pipelines/oneshot/dual_upload_mixing.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Dual Upload Mixer description: Mix two uploaded Ogg/Opus tracks and return Opus/WebM mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: audio nodes: # ============================================================ # INPUTS: Two uploaded audio tracks diff --git a/samples/pipelines/oneshot/gain_filter_rust.yml b/samples/pipelines/oneshot/gain_filter_rust.yml index 70293187..b81121a0 100644 --- a/samples/pipelines/oneshot/gain_filter_rust.yml +++ b/samples/pipelines/oneshot/gain_filter_rust.yml @@ -1,6 +1,12 @@ name: Gain Filter (Rust WASM) description: Applies a gain filter using the Rust WASM plugin mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio steps: - kind: streamkit::http_input - kind: containers::ogg::demuxer diff --git a/samples/pipelines/oneshot/kokoro-tts.yml b/samples/pipelines/oneshot/kokoro-tts.yml index 0c88367b..f438c802 100644 --- a/samples/pipelines/oneshot/kokoro-tts.yml +++ b/samples/pipelines/oneshot/kokoro-tts.yml @@ -1,6 +1,12 @@ name: Text-to-Speech (Kokoro) description: Synthesizes speech from text using Kokoro mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/matcha-tts.yml b/samples/pipelines/oneshot/matcha-tts.yml index 517743df..f8a01a8a 100644 --- a/samples/pipelines/oneshot/matcha-tts.yml +++ b/samples/pipelines/oneshot/matcha-tts.yml @@ -5,6 +5,12 @@ name: Text-to-Speech (Matcha) description: Synthesizes speech from text using Matcha mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/mixing.yml b/samples/pipelines/oneshot/mixing.yml index 2b98df29..5b42e031 100644 --- a/samples/pipelines/oneshot/mixing.yml +++ b/samples/pipelines/oneshot/mixing.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Audio Mixing (Upload + Music Track) description: Mixes uploaded audio with a built-in music track and returns Opus/WebM mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: audio nodes: # ============================================================ # INPUTS: HTTP upload + local music file diff --git a/samples/pipelines/oneshot/piper-tts.yml b/samples/pipelines/oneshot/piper-tts.yml index 4f9e0c48..40692f4f 100644 --- a/samples/pipelines/oneshot/piper-tts.yml +++ b/samples/pipelines/oneshot/piper-tts.yml @@ -5,6 +5,12 @@ name: Text-to-Speech (Piper) description: Synthesizes speech from text using Piper mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/pocket-tts-voice-clone.yml b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml index 095714e9..10611131 100644 --- a/samples/pipelines/oneshot/pocket-tts-voice-clone.yml +++ b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml @@ -2,11 +2,21 @@ # # SPDX-License-Identifier: MPL-2.0 -# skit:input_formats=voice:wav - name: Text-to-Speech (Pocket TTS Voice Clone) description: Synthesizes speech from text using Pocket TTS with a voice prompt WAV mode: oneshot +client: + input: + type: file_upload + field_hints: + voice: + type: file + accept: "audio/wav" + text: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio nodes: uploads: kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/pocket-tts.yml b/samples/pipelines/oneshot/pocket-tts.yml index ae088dd7..0981de9f 100644 --- a/samples/pipelines/oneshot/pocket-tts.yml +++ b/samples/pipelines/oneshot/pocket-tts.yml @@ -5,6 +5,12 @@ name: Text-to-Speech (Pocket TTS) description: Synthesizes speech from text using Pocket TTS mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/sensevoice-stt.yml b/samples/pipelines/oneshot/sensevoice-stt.yml index 25b197e8..8f27d51b 100644 --- a/samples/pipelines/oneshot/sensevoice-stt.yml +++ b/samples/pipelines/oneshot/sensevoice-stt.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Speech-to-Text (SenseVoice) description: Transcribes speech in multiple languages using SenseVoice mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: transcription steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/speech_to_text.yml b/samples/pipelines/oneshot/speech_to_text.yml index f772a676..6c2441a8 100644 --- a/samples/pipelines/oneshot/speech_to_text.yml +++ b/samples/pipelines/oneshot/speech_to_text.yml @@ -1,9 +1,14 @@ -# -# skit:input_asset_tags=speech - name: Speech-to-Text (Whisper) description: Transcribes speech to text using Whisper mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: transcription steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/speech_to_text_translate.yml b/samples/pipelines/oneshot/speech_to_text_translate.yml index e0eede96..c4a29be0 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate.yml @@ -2,12 +2,17 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - name: Speech Translation (English → Spanish) description: Translates English speech into Spanish speech mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: audio steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml index 11c96746..3eb28a78 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml @@ -2,9 +2,6 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - # Speech Translation using Helsinki-NLP OPUS-MT (Apache 2.0 licensed) # # This pipeline is an alternative to speech_to_text_translate.yml that uses @@ -18,6 +15,14 @@ name: Speech Translation (English → Spanish) - Helsinki description: Translates English speech into Spanish speech using Apache 2.0 licensed Helsinki OPUS-MT mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: audio steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/supertonic-tts.yml b/samples/pipelines/oneshot/supertonic-tts.yml index ca793cd9..1dcc8019 100644 --- a/samples/pipelines/oneshot/supertonic-tts.yml +++ b/samples/pipelines/oneshot/supertonic-tts.yml @@ -1,6 +1,12 @@ name: Text-to-Speech (Supertonic) description: Synthesizes speech from text using Supertonic (5 languages, 10 voice styles) mode: oneshot +client: + input: + type: text + placeholder: "Enter text to synthesize" + output: + type: audio steps: - kind: streamkit::http_input - kind: core::text_chunker diff --git a/samples/pipelines/oneshot/useless-facts-tts.yml b/samples/pipelines/oneshot/useless-facts-tts.yml index ee252c5b..91525dbc 100644 --- a/samples/pipelines/oneshot/useless-facts-tts.yml +++ b/samples/pipelines/oneshot/useless-facts-tts.yml @@ -5,6 +5,11 @@ name: Text-to-Speech (Useless Facts) description: Fetches a random fact and reads it out loud using Kokoro mode: oneshot +client: + input: + type: trigger + output: + type: audio steps: # HTTP input (body can be empty, just triggers the pipeline) - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/vad-demo.yml b/samples/pipelines/oneshot/vad-demo.yml index 1d9b4bde..0833504e 100644 --- a/samples/pipelines/oneshot/vad-demo.yml +++ b/samples/pipelines/oneshot/vad-demo.yml @@ -2,12 +2,17 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - name: Voice Activity Detection description: Detects voice activity and outputs events as JSON mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: json steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/vad-filtered-stt.yml b/samples/pipelines/oneshot/vad-filtered-stt.yml index e3395b5f..6a8af170 100644 --- a/samples/pipelines/oneshot/vad-filtered-stt.yml +++ b/samples/pipelines/oneshot/vad-filtered-stt.yml @@ -2,12 +2,17 @@ # # SPDX-License-Identifier: MPL-2.0 -# -# skit:input_asset_tags=speech - name: Speech-to-Text (Whisper, VAD-Filtered) description: Filters silence with VAD before transcribing with Whisper mode: oneshot +client: + input: + type: file_upload + accept: "audio/*" + asset_tags: + - speech + output: + type: transcription steps: - kind: streamkit::http_input diff --git a/samples/pipelines/oneshot/video_colorbars.yml b/samples/pipelines/oneshot/video_colorbars.yml index de72e242..24bd1d5e 100644 --- a/samples/pipelines/oneshot/video_colorbars.yml +++ b/samples/pipelines/oneshot/video_colorbars.yml @@ -5,6 +5,11 @@ name: Video Color Bars (VP9/WebM) description: Generates SMPTE color bars, encodes to VP9, and outputs a WebM file (5 seconds) mode: oneshot +client: + input: + type: none + output: + type: video nodes: colorbars: diff --git a/samples/pipelines/oneshot/video_compositor_demo.yml b/samples/pipelines/oneshot/video_compositor_demo.yml index e41beefd..95e1b277 100644 --- a/samples/pipelines/oneshot/video_compositor_demo.yml +++ b/samples/pipelines/oneshot/video_compositor_demo.yml @@ -16,6 +16,11 @@ name: Video Compositor Demo description: Composites two colorbars sources (main + PiP) with a text overlay through the compositor node, encodes to VP9, and streams a WebM via http_output with real-time pacing (3 seconds) mode: oneshot +client: + input: + type: none + output: + type: video nodes: colorbars_bg: From 56957040e35d65552bc1f8b862f648cf5f6cb36c Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:19:11 +0000 Subject: [PATCH 03/20] feat: wire up dynamic UI to read from client section (Phase 3) - Replace extractMoqPeerSettings() internals with client-section reads, removing 6 internal graph-scanning helpers (~170 lines deleted) - Update useMonitorPreview to read pipeline.client instead of scanning the compiled node graph for transport::moq::peer nodes - Rewrite moqPeerSettings tests for declarative client-section behavior - Add client: null to Pipeline objects in test files for type compliance Co-Authored-By: Claudio Costa --- ui/src/hooks/useMonitorPreview.ts | 54 +-- ui/src/services/websocket.test.ts | 1 + ui/src/stores/sessionStore.edge-cases.test.ts | 9 + ui/src/stores/sessionStore.test.ts | 10 + ui/src/utils/moqPeerSettings.test.ts | 348 ++++++------------ ui/src/utils/moqPeerSettings.ts | 223 +---------- 6 files changed, 161 insertions(+), 484 deletions(-) diff --git a/ui/src/hooks/useMonitorPreview.ts b/ui/src/hooks/useMonitorPreview.ts index 0a6dc39b..d57f5283 100644 --- a/ui/src/hooks/useMonitorPreview.ts +++ b/ui/src/hooks/useMonitorPreview.ts @@ -69,8 +69,8 @@ export function useMonitorPreview( } }, [selectedSessionId, previewStatus, previewDisconnect]); - // Extract MoQ peer settings from the selected session's pipeline so the - // preview connects to the correct gateway path and output broadcast. + // Read MoQ peer settings from the pipeline's declarative client section + // instead of scanning the compiled node graph. const handleStartPreview = useCallback(async () => { // Configure for watch-only mode (no publish/mic) previewSetEnablePublish(false); @@ -79,46 +79,24 @@ export function useMonitorPreview( await previewLoadConfig(); } - // Extract gateway_path and output_broadcast from the pipeline's moq_peer node - let moqNodeName: string | undefined; - const moqNode = pipeline - ? Object.entries(pipeline.nodes).find( - ([, n]) => n.kind === 'transport::moq::peer' && n.params - ) - : undefined; - if (moqNode) { - moqNodeName = moqNode[0]; - const params = moqNode[1].params as Record; - const gatewayPath = params.gateway_path as string | undefined; - const outputBroadcast = params.output_broadcast as string | undefined; - // Read serverUrl at call-time via getState() rather than via a - // hook selector — this is a standard Zustand pattern for values - // that should be fresh when the callback fires, not stale from - // the last render. - const currentUrl = useStreamStore.getState().serverUrl; - if (gatewayPath && currentUrl) { - previewSetServerUrl(updateUrlPath(currentUrl, gatewayPath)); + // Read gateway_path and output_broadcast from the pipeline's client section. + const client = pipeline?.client ?? null; + if (client) { + if (client.gateway_path) { + const currentUrl = useStreamStore.getState().serverUrl; + if (currentUrl) { + previewSetServerUrl(updateUrlPath(currentUrl, client.gateway_path)); + } } - if (outputBroadcast) { - previewSetOutputBroadcast(outputBroadcast); + if (client.watch?.broadcast) { + previewSetOutputBroadcast(client.watch.broadcast); } } - // Detect which media types the pipeline outputs by checking the kinds of - // nodes connected to the moq_peer's input pins. - let outputsAudio = true; - let outputsVideo = true; - if (pipeline && moqNodeName) { - outputsAudio = false; - outputsVideo = false; - for (const conn of pipeline.connections) { - if (conn.to_node !== moqNodeName) continue; - const sourceNode = pipeline.nodes[conn.from_node]; - if (!sourceNode?.kind) continue; - if (sourceNode.kind.startsWith('audio::')) outputsAudio = true; - else if (sourceNode.kind.startsWith('video::')) outputsVideo = true; - } - } + // Media types default to both enabled unless the client section + // explicitly declares which types the pipeline outputs. + const outputsAudio = client?.watch?.audio ?? true; + const outputsVideo = client?.watch?.video ?? true; previewSetPipelineOutputTypes(outputsAudio, outputsVideo); await previewConnect(); diff --git a/ui/src/services/websocket.test.ts b/ui/src/services/websocket.test.ts index 094f2c5f..3e8b7258 100644 --- a/ui/src/services/websocket.test.ts +++ b/ui/src/services/websocket.test.ts @@ -325,6 +325,7 @@ describe('WebSocketService', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }, diff --git a/ui/src/stores/sessionStore.edge-cases.test.ts b/ui/src/stores/sessionStore.edge-cases.test.ts index 03593945..cd89ff21 100644 --- a/ui/src/stores/sessionStore.edge-cases.test.ts +++ b/ui/src/stores/sessionStore.edge-cases.test.ts @@ -50,6 +50,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'core::passthrough', params: {}, state: 'Initializing' } }, connections: [], }; @@ -57,6 +58,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-2': { kind: 'core::gain', params: { gain: 1.0 }, state: 'Initializing' } }, connections: [], }; @@ -98,6 +100,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'core::passthrough', @@ -236,6 +239,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { compositor: { kind: 'video::compositor', @@ -266,6 +270,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], view_data: { nodeA: { key: 'original' } }, @@ -276,6 +281,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], view_data: { nodeB: { key: 'new' } }, @@ -292,6 +298,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -310,6 +317,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic' as const, + client: null, nodes: {}, connections: [], view_data: { comp: { layers: { in_0: { x: 10 } } } }, @@ -330,6 +338,7 @@ describe('sessionStore edge cases', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'core::passthrough', params: {}, state: 'Initializing' }, }, diff --git a/ui/src/stores/sessionStore.test.ts b/ui/src/stores/sessionStore.test.ts index ce5f86d9..8b5ea854 100644 --- a/ui/src/stores/sessionStore.test.ts +++ b/ui/src/stores/sessionStore.test.ts @@ -107,6 +107,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -129,6 +130,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -142,6 +144,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -179,6 +182,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -200,6 +204,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -223,6 +228,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: { 'node-1': { kind: 'gain', @@ -254,6 +260,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -279,6 +286,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [ { @@ -338,6 +346,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; @@ -362,6 +371,7 @@ describe('sessionStore', () => { name: null, description: null, mode: 'dynamic', + client: null, nodes: {}, connections: [], }; diff --git a/ui/src/utils/moqPeerSettings.test.ts b/ui/src/utils/moqPeerSettings.test.ts index e49678a8..0853d430 100644 --- a/ui/src/utils/moqPeerSettings.test.ts +++ b/ui/src/utils/moqPeerSettings.test.ts @@ -7,7 +7,7 @@ import { describe, expect, it } from 'vitest'; import { extractMoqPeerSettings } from './moqPeerSettings'; // --------------------------------------------------------------------------- -// extractMoqPeerSettings — basic parsing +// extractMoqPeerSettings — reads the declarative `client` section // --------------------------------------------------------------------------- describe('extractMoqPeerSettings', () => { @@ -15,311 +15,177 @@ describe('extractMoqPeerSettings', () => { expect(extractMoqPeerSettings('')).toBeNull(); }); - it('should return null for YAML without nodes', () => { + it('should return null for YAML without client section', () => { expect(extractMoqPeerSettings('name: test')).toBeNull(); }); - it('should return null when no moq_peer node exists', () => { - const yaml = ` -nodes: - gain: - kind: audio::gain - params: - gain: 1.0 -`; - expect(extractMoqPeerSettings(yaml)).toBeNull(); - }); - it('should return null for invalid YAML', () => { expect(extractMoqPeerSettings('{{{')).toBeNull(); }); - it('should return null when moq_peer has no params', () => { + it('should return null for oneshot client (input/output only)', () => { const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio nodes: - moq_peer: - kind: transport::moq::peer + gain: + kind: audio::gain `; expect(extractMoqPeerSettings(yaml)).toBeNull(); }); - it('should extract basic moq_peer settings', () => { + it('should extract gateway-based settings with publish and watch', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output +client: + gateway_path: /moq/echo + publish: + broadcast: echo-demo + audio: true + video: false + watch: + broadcast: echo-demo + audio: true + video: false `; const result = extractMoqPeerSettings(yaml); expect(result).not.toBeNull(); - expect(result!.gatewayPath).toBe('/moq'); - expect(result!.inputBroadcast).toBe('input'); - expect(result!.outputBroadcast).toBe('output'); + expect(result!.gatewayPath).toBe('/moq/echo'); + expect(result!.inputBroadcast).toBe('echo-demo'); + expect(result!.outputBroadcast).toBe('echo-demo'); expect(result!.hasInputBroadcast).toBe(true); - }); - - it('should report hasInputBroadcast false when input_broadcast is absent', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output -`; - const result = extractMoqPeerSettings(yaml); - expect(result).not.toBeNull(); - expect(result!.hasInputBroadcast).toBe(false); - }); -}); - -// --------------------------------------------------------------------------- -// detectPeerInputMediaTypes (exercised through extractMoqPeerSettings) -// --------------------------------------------------------------------------- - -describe('detectPeerInputMediaTypes (via extractMoqPeerSettings)', () => { - it('should detect audio input (bare peer reference)', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - opus_decoder: - kind: audio::opus::decoder - needs: moq_peer -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.needsAudioInput).toBe(true); - expect(result!.needsVideoInput).toBe(false); - }); - - it('should detect audio input (explicit .out pin)', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - opus_decoder: - kind: audio::opus::decoder - needs: moq_peer.out -`; - const result = extractMoqPeerSettings(yaml); expect(result!.needsAudioInput).toBe(true); expect(result!.needsVideoInput).toBe(false); + expect(result!.outputsAudio).toBe(true); + expect(result!.outputsVideo).toBe(false); }); - it('should detect video input (.out_1 pin)', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - vp9_decoder: - kind: video::vp9::decoder - needs: moq_peer.out_1 -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.needsAudioInput).toBe(false); - expect(result!.needsVideoInput).toBe(true); - }); - - it('should detect both audio and video inputs', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - opus_decoder: - kind: audio::opus::decoder - needs: moq_peer.out - vp9_decoder: - kind: video::vp9::decoder - needs: moq_peer.out_1 -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.needsAudioInput).toBe(true); - expect(result!.needsVideoInput).toBe(true); - }); - - it('should handle needs as array', () => { + it('should extract relay-based settings', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - mixer: - kind: audio::mixer - needs: - - moq_peer - - other_source +client: + relay_url: "https://relay.example.com" + publish: + broadcast: input-stream + audio: true + video: true + watch: + broadcast: output-stream + audio: true + video: false `; const result = extractMoqPeerSettings(yaml); + expect(result).not.toBeNull(); + expect(result!.relayUrl).toBe('https://relay.example.com'); + expect(result!.gatewayPath).toBeUndefined(); + expect(result!.inputBroadcast).toBe('input-stream'); + expect(result!.outputBroadcast).toBe('output-stream'); + expect(result!.hasInputBroadcast).toBe(true); expect(result!.needsAudioInput).toBe(true); - }); - - it('should handle needs as map (record)', () => { - const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - compositor: - kind: video::compositor - needs: - in: colorbars - in_1: moq_peer.out_1 -`; - const result = extractMoqPeerSettings(yaml); expect(result!.needsVideoInput).toBe(true); + expect(result!.outputsAudio).toBe(true); + expect(result!.outputsVideo).toBe(false); }); - it('should report no inputs when no downstream nodes reference the peer', () => { + it('should report hasInputBroadcast false when publish is absent', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - gain: - kind: audio::gain - needs: some_other_node +client: + gateway_path: /moq/colorbars + watch: + broadcast: colorbars + audio: false + video: true `; const result = extractMoqPeerSettings(yaml); + expect(result).not.toBeNull(); + expect(result!.hasInputBroadcast).toBe(false); expect(result!.needsAudioInput).toBe(false); expect(result!.needsVideoInput).toBe(false); + expect(result!.outputsAudio).toBe(false); + expect(result!.outputsVideo).toBe(true); }); -}); -// --------------------------------------------------------------------------- -// detectPeerOutputMediaTypes (exercised through extractMoqPeerSettings) -// --------------------------------------------------------------------------- - -describe('detectPeerOutputMediaTypes (via extractMoqPeerSettings)', () => { - it('should detect audio output when moq_peer needs an audio:: node', () => { + it('should detect audio+video publish and watch', () => { const yaml = ` -nodes: - opus_encoder: - kind: audio::opus::encoder - needs: gain - gain: - kind: audio::gain - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: opus_encoder +client: + gateway_path: /moq/av + publish: + broadcast: av-input + audio: true + video: true + watch: + broadcast: av-output + audio: true + video: true `; const result = extractMoqPeerSettings(yaml); + expect(result!.needsAudioInput).toBe(true); + expect(result!.needsVideoInput).toBe(true); expect(result!.outputsAudio).toBe(true); - expect(result!.outputsVideo).toBe(false); - }); - - it('should detect video output when moq_peer needs a video:: node', () => { - const yaml = ` -nodes: - vp9_encoder: - kind: video::vp9::encoder - needs: pixel_convert - pixel_convert: - kind: video::pixel_convert - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: vp9_encoder -`; - const result = extractMoqPeerSettings(yaml); - expect(result!.outputsAudio).toBe(false); expect(result!.outputsVideo).toBe(true); }); - it('should detect both audio and video outputs', () => { + it('should handle watch-only pipeline (no publish)', () => { const yaml = ` -nodes: - opus_encoder: - kind: audio::opus::encoder - vp9_encoder: - kind: video::vp9::encoder - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: - in: opus_encoder - in_1: vp9_encoder +client: + gateway_path: /moq/output + watch: + broadcast: output-stream + audio: true + video: true `; const result = extractMoqPeerSettings(yaml); + expect(result).not.toBeNull(); + expect(result!.hasInputBroadcast).toBe(false); + expect(result!.needsAudioInput).toBe(false); + expect(result!.needsVideoInput).toBe(false); expect(result!.outputsAudio).toBe(true); expect(result!.outputsVideo).toBe(true); }); - it('should report no outputs when moq_peer has no needs', () => { + it('should return settings when only gateway_path is present', () => { const yaml = ` -nodes: - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output +client: + gateway_path: /moq/peer `; const result = extractMoqPeerSettings(yaml); + expect(result).not.toBeNull(); + expect(result!.gatewayPath).toBe('/moq/peer'); + expect(result!.hasInputBroadcast).toBe(false); expect(result!.outputsAudio).toBe(false); expect(result!.outputsVideo).toBe(false); }); - it('should handle dotted pin references in moq_peer needs', () => { + it('should return settings when only relay_url is present', () => { const yaml = ` -nodes: - opus_encoder: - kind: audio::opus::encoder - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - input_broadcast: input - output_broadcast: output - needs: opus_encoder.out +client: + relay_url: "https://relay.example.com" `; const result = extractMoqPeerSettings(yaml); - // "opus_encoder.out" → nodeName is "opus_encoder" → kind is audio:: → outputsAudio - expect(result!.outputsAudio).toBe(true); + expect(result).not.toBeNull(); + expect(result!.relayUrl).toBe('https://relay.example.com'); + expect(result!.hasInputBroadcast).toBe(false); }); - it('should ignore upstream nodes without a kind', () => { + it('should handle audio-only publish', () => { const yaml = ` -nodes: - unknown_node: {} - moq_peer: - kind: transport::moq::peer - params: - gateway_path: /moq - output_broadcast: output - needs: unknown_node +client: + gateway_path: /moq/audio + publish: + broadcast: mic + audio: true + video: false + watch: + broadcast: processed + audio: true + video: false `; const result = extractMoqPeerSettings(yaml); - expect(result!.outputsAudio).toBe(false); + expect(result!.needsAudioInput).toBe(true); + expect(result!.needsVideoInput).toBe(false); + expect(result!.outputsAudio).toBe(true); expect(result!.outputsVideo).toBe(false); }); }); diff --git a/ui/src/utils/moqPeerSettings.ts b/ui/src/utils/moqPeerSettings.ts index 99ec803d..893ba8f1 100644 --- a/ui/src/utils/moqPeerSettings.ts +++ b/ui/src/utils/moqPeerSettings.ts @@ -4,6 +4,10 @@ import { load } from 'js-yaml'; +import type { ClientSection } from '@/types/types'; + +import { deriveSettingsFromClient } from './clientSection'; + export interface MoqPeerSettings { gatewayPath?: string; /** Direct relay URL from publisher/subscriber `url` param (external relay pattern). */ @@ -22,222 +26,31 @@ export interface MoqPeerSettings { outputsVideo: boolean; } -type NeedsValue = string | string[] | Record; - -type ParsedNode = { - kind?: string; - params?: { - gateway_path?: string; - input_broadcast?: string; - output_broadcast?: string; - url?: string; - broadcast?: string; - audio?: boolean; - video?: boolean; - [key: string]: unknown; - }; - needs?: NeedsValue; -}; - -type ParsedYaml = { - nodes?: Record; -}; - -/** - * Collects all dependency references from a node's `needs` field as flat strings. - */ -function collectNeedsRefs(needs: NeedsValue | undefined): string[] { - if (!needs) return []; - if (typeof needs === 'string') return [needs]; - if (Array.isArray(needs)) return needs.filter((v): v is string => typeof v === 'string'); - // Record (map variant) — values are the dependency refs - return Object.values(needs).filter((v): v is string => typeof v === 'string'); -} - /** - * Scans all nodes in the pipeline to detect which media types downstream nodes - * consume from the moq_peer's output pins. + * Extracts MoQ peer settings from a pipeline YAML string by reading the + * declarative `client` section. * - * The `transport::moq::peer` node exposes two fixed output pins: - * - `out` → audio (Opus-encoded) - * - `out_1` → video (VP9-encoded) - * - * This is a stable naming convention enforced by the backend node - * definition (see `MoqPeerNode::output_pins` in `crates/nodes/src/transport/moq/peer.rs`). - * A bare reference to the peer name (e.g. `moq_peer`) is equivalent to `moq_peer.out`. - */ -function detectPeerInputMediaTypes( - peerName: string, - nodes: Record -): { needsAudio: boolean; needsVideo: boolean } { - let needsAudio = false; - let needsVideo = false; - - for (const [nodeName, nodeConfig] of Object.entries(nodes)) { - if (nodeName === peerName) continue; - for (const ref of collectNeedsRefs(nodeConfig.needs)) { - if (ref === peerName || ref === `${peerName}.out`) { - needsAudio = true; - } else if (ref === `${peerName}.out_1`) { - needsVideo = true; - } - } - } - - return { needsAudio, needsVideo }; -} - -/** - * Detects what media types the moq_peer outputs to subscribers by looking at - * which upstream nodes are connected to its inputs. The node `kind` prefix - * (`audio::` or `video::`) determines the media type. - */ -function detectPeerOutputMediaTypes( - peerName: string, - nodes: Record -): { outputsAudio: boolean; outputsVideo: boolean } { - const peerNode = nodes[peerName]; - if (!peerNode) return { outputsAudio: false, outputsVideo: false }; - - let outputsAudio = false; - let outputsVideo = false; - - for (const ref of collectNeedsRefs(peerNode.needs)) { - const nodeName = ref.split('.')[0]; - const sourceNode = nodes[nodeName]; - if (!sourceNode?.kind) continue; - - if (sourceNode.kind.startsWith('audio::')) { - outputsAudio = true; - } else if (sourceNode.kind.startsWith('video::')) { - outputsVideo = true; - } - } - - return { outputsAudio, outputsVideo }; -} - -/** Finds the first subscriber and publisher nodes in the pipeline. */ -function findPubSubNodes(nodes: Record): { - subscriberName: string | null; - subscriberConfig: ParsedNode | null; - publisherConfig: ParsedNode | null; -} { - let subscriberName: string | null = null; - let subscriberConfig: ParsedNode | null = null; - let publisherConfig: ParsedNode | null = null; - - for (const [name, nodeConfig] of Object.entries(nodes)) { - if (nodeConfig.kind === 'transport::moq::subscriber') { - subscriberName = name; - subscriberConfig = nodeConfig; - } else if (nodeConfig.kind === 'transport::moq::publisher') { - publisherConfig = nodeConfig; - } - } - - return { subscriberName, subscriberConfig, publisherConfig }; -} - -/** - * Detects which media types downstream nodes consume from a subscriber node - * by scanning `needs` references across all nodes. - */ -function detectSubscriberInputMediaTypes( - subscriberName: string, - nodes: Record -): { needsAudio: boolean; needsVideo: boolean } { - let needsAudio = false; - let needsVideo = false; - - for (const nodeConfig of Object.values(nodes)) { - for (const ref of collectNeedsRefs(nodeConfig.needs)) { - if (ref === subscriberName || ref === `${subscriberName}.out`) { - needsAudio = true; - } else if (ref.startsWith(`${subscriberName}.`) && ref.includes('video')) { - needsVideo = true; - } - } - } - - return { needsAudio, needsVideo }; -} - -/** - * Detects media types for pipelines using separate `transport::moq::publisher` - * and `transport::moq::subscriber` nodes (external relay pattern). - */ -function extractPubSubSettings(nodes: Record): MoqPeerSettings | null { - const { subscriberName, subscriberConfig, publisherConfig } = findPubSubNodes(nodes); - if (!subscriberConfig && !publisherConfig) return null; - - const subParams = subscriberConfig?.params; - const pubParams = publisherConfig?.params; - - const inputMedia = - subscriberName != null - ? detectSubscriberInputMediaTypes(subscriberName, nodes) - : { needsAudio: false, needsVideo: false }; - - return { - relayUrl: subParams?.url ?? pubParams?.url, - inputBroadcast: subParams?.broadcast, - outputBroadcast: pubParams?.broadcast, - hasInputBroadcast: Boolean(subParams?.broadcast), - needsAudioInput: inputMedia.needsAudio, - needsVideoInput: inputMedia.needsVideo, - outputsAudio: pubParams?.audio === true, - outputsVideo: pubParams?.video === true, - }; -} - -/** - * Extracts moq_peer settings from a pipeline YAML string. - * Looks for `transport::moq::peer` nodes first (gateway pattern), then falls - * back to separate `transport::moq::publisher`/`subscriber` nodes (external - * relay pattern). + * Returns settings only when the client section declares dynamic transport + * configuration (gateway_path, relay_url, publish, or watch). Oneshot + * pipelines (input/output only) return null. * * @param yamlContent - The YAML string to parse - * @returns MoqPeerSettings if MoQ transport nodes are found, null otherwise + * @returns MoqPeerSettings if the client section declares MoQ transport, null otherwise */ export function extractMoqPeerSettings(yamlContent: string): MoqPeerSettings | null { try { - const parsed = load(yamlContent) as ParsedYaml; + const parsed = load(yamlContent) as Record | null; + if (!parsed || typeof parsed !== 'object') return null; - if (!parsed || typeof parsed !== 'object' || !parsed.nodes) { - return null; - } - - // Find the first node with kind 'transport::moq::peer' - let peerNodeName: string | null = null; - let peerNodeConfig: ParsedNode | null = null; - for (const [name, nodeConfig] of Object.entries(parsed.nodes)) { - if (nodeConfig.kind === 'transport::moq::peer') { - peerNodeName = name; - peerNodeConfig = nodeConfig; - break; - } - } - - // Gateway pattern: transport::moq::peer - if (peerNodeName && peerNodeConfig?.params) { - const { needsAudio, needsVideo } = detectPeerInputMediaTypes(peerNodeName, parsed.nodes); - const { outputsAudio, outputsVideo } = detectPeerOutputMediaTypes(peerNodeName, parsed.nodes); + const client = parsed.client as ClientSection | undefined; + if (!client) return null; - return { - gatewayPath: peerNodeConfig.params.gateway_path, - inputBroadcast: peerNodeConfig.params.input_broadcast, - outputBroadcast: peerNodeConfig.params.output_broadcast, - hasInputBroadcast: Boolean(peerNodeConfig.params.input_broadcast), - needsAudioInput: needsAudio, - needsVideoInput: needsVideo, - outputsAudio, - outputsVideo, - }; + // Only return settings for dynamic pipelines that declare MoQ transport. + if (!client.gateway_path && !client.relay_url && !client.publish && !client.watch) { + return null; } - // External relay pattern: separate publisher/subscriber nodes - return extractPubSubSettings(parsed.nodes); + return deriveSettingsFromClient(client); } catch { return null; } From a4c5ba9a149d48276c6bb44d48b051ab69603473 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:25:49 +0000 Subject: [PATCH 04/20] =?UTF-8?q?phase=204:=20wire=20up=20oneshot=20UI=20?= =?UTF-8?q?=E2=80=94=20replace=20ConvertView=20heuristics=20with=20client?= =?UTF-8?q?=20section=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete ~10 heuristic functions from ConvertView.tsx: checkIfTranscriptionPipeline, checkIfTTSPipeline, checkIfNoInputPipeline, checkIfVideoPipeline, FORMAT_ACCEPT_MAP, detectInputFormatSpec, resolveFormatsForField, formatHintForField, detectExpectedFormats, detectInputAssetTags - Derive isTranscription/isTTS/isNoInput/isVideo from client section useMemo - Replace format detection with parseAcceptToFormats(client.input.accept) - Add getFieldHint callback using client.input.field_hints - Remove isTranscriptionPipeline/isTTSPipeline/isNoInputPipeline state from useConvertViewState.ts (now derived values) - Update moqPeerSettings.ts to use parseClientFromYaml from clientSection.ts - Move assetMatchesTag outside the component - Add 14 new tests for parseAcceptToFormats and parseClientFromYaml - All 280 tests pass, lint clean Co-Authored-By: Claudio Costa --- ui/src/hooks/useConvertViewState.ts | 9 - ui/src/utils/clientSection.test.ts | 115 +++++++- ui/src/utils/moqPeerSettings.ts | 25 +- ui/src/views/ConvertView.tsx | 397 ++++------------------------ 4 files changed, 178 insertions(+), 368 deletions(-) diff --git a/ui/src/hooks/useConvertViewState.ts b/ui/src/hooks/useConvertViewState.ts index 839ba034..7f201b44 100644 --- a/ui/src/hooks/useConvertViewState.ts +++ b/ui/src/hooks/useConvertViewState.ts @@ -30,9 +30,6 @@ export function useConvertViewState() { // Pipeline state const [pipelineYaml, setPipelineYaml] = useState(''); const [selectedTemplateId, setSelectedTemplateId] = useState(''); - const [isTranscriptionPipeline, setIsTranscriptionPipeline] = useState(false); - const [isTTSPipeline, setIsTTSPipeline] = useState(false); - const [isNoInputPipeline, setIsNoInputPipeline] = useState(false); const [textInput, setTextInput] = useState(''); // Conversion state @@ -73,12 +70,6 @@ export function useConvertViewState() { setPipelineYaml, selectedTemplateId, setSelectedTemplateId, - isTranscriptionPipeline, - setIsTranscriptionPipeline, - isTTSPipeline, - setIsTTSPipeline, - isNoInputPipeline, - setIsNoInputPipeline, textInput, setTextInput, diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts index f4c56b1c..fa6bc3df 100644 --- a/ui/src/utils/clientSection.test.ts +++ b/ui/src/utils/clientSection.test.ts @@ -6,7 +6,12 @@ import { describe, expect, it } from 'vitest'; import type { ClientSection } from '@/types/types'; -import { deriveSettingsFromClient, extractClientSection } from './clientSection'; +import { + deriveSettingsFromClient, + extractClientSection, + parseAcceptToFormats, + parseClientFromYaml, +} from './clientSection'; describe('extractClientSection', () => { it('returns null for null pipeline', () => { @@ -143,3 +148,111 @@ describe('deriveSettingsFromClient', () => { }); }); }); + +describe('parseAcceptToFormats', () => { + it('returns null for null/undefined', () => { + expect(parseAcceptToFormats(null)).toBeNull(); + expect(parseAcceptToFormats(undefined)).toBeNull(); + }); + + it('returns null for empty string', () => { + expect(parseAcceptToFormats('')).toBeNull(); + }); + + it('returns null for wildcard accept', () => { + expect(parseAcceptToFormats('audio/*')).toBeNull(); + expect(parseAcceptToFormats('*/*')).toBeNull(); + }); + + it('parses dot-prefixed extensions', () => { + expect(parseAcceptToFormats('.ogg,.opus')).toEqual(['ogg', 'opus']); + }); + + it('parses MIME types', () => { + expect(parseAcceptToFormats('audio/wav,audio/ogg')).toEqual(['wav', 'ogg']); + }); + + it('parses mixed extensions and MIME types', () => { + expect(parseAcceptToFormats('audio/wav,.wav')).toEqual(['wav', 'wav']); + }); + + it('handles bare format names', () => { + expect(parseAcceptToFormats('ogg,opus')).toEqual(['ogg', 'opus']); + }); + + it('lowercases all formats', () => { + expect(parseAcceptToFormats('.OGG,.OPUS')).toEqual(['ogg', 'opus']); + }); + + it('trims whitespace', () => { + expect(parseAcceptToFormats(' .ogg , .opus ')).toEqual(['ogg', 'opus']); + }); +}); + +describe('parseClientFromYaml', () => { + it('returns null for empty YAML', () => { + expect(parseClientFromYaml('')).toBeNull(); + }); + + it('returns null for YAML without client section', () => { + const yaml = ` +steps: + - kind: ogg::demuxer +`; + expect(parseClientFromYaml(yaml)).toBeNull(); + }); + + it('parses a full client section from YAML', () => { + const yaml = ` +client: + input: + type: file_upload + accept: ".ogg,.opus" + asset_tags: + - speech + field_hints: + voice: + type: file + accept: "audio/wav,.wav" + output: + type: transcription +steps: + - kind: ogg::demuxer +`; + const client = parseClientFromYaml(yaml); + expect(client).not.toBeNull(); + expect(client?.input?.type).toBe('file_upload'); + expect(client?.input?.accept).toBe('.ogg,.opus'); + expect(client?.input?.asset_tags).toEqual(['speech']); + expect(client?.input?.field_hints?.voice?.type).toBe('file'); + expect(client?.input?.field_hints?.voice?.accept).toBe('audio/wav,.wav'); + expect(client?.output?.type).toBe('transcription'); + }); + + it('parses a dynamic pipeline client section', () => { + const yaml = ` +client: + gateway_path: /moq/compositor + publish: + broadcast: camera-feed + audio: true + video: true + watch: + broadcast: composited-output + audio: true + video: true +nodes: + pub: + kind: moq::subscriber +`; + const client = parseClientFromYaml(yaml); + expect(client).not.toBeNull(); + expect(client?.gateway_path).toBe('/moq/compositor'); + expect(client?.publish?.broadcast).toBe('camera-feed'); + expect(client?.watch?.broadcast).toBe('composited-output'); + }); + + it('returns null for invalid YAML', () => { + expect(parseClientFromYaml('{{invalid')).toBeNull(); + }); +}); diff --git a/ui/src/utils/moqPeerSettings.ts b/ui/src/utils/moqPeerSettings.ts index 893ba8f1..a60ce1a1 100644 --- a/ui/src/utils/moqPeerSettings.ts +++ b/ui/src/utils/moqPeerSettings.ts @@ -2,11 +2,7 @@ // // SPDX-License-Identifier: MPL-2.0 -import { load } from 'js-yaml'; - -import type { ClientSection } from '@/types/types'; - -import { deriveSettingsFromClient } from './clientSection'; +import { deriveSettingsFromClient, parseClientFromYaml } from './clientSection'; export interface MoqPeerSettings { gatewayPath?: string; @@ -38,22 +34,15 @@ export interface MoqPeerSettings { * @returns MoqPeerSettings if the client section declares MoQ transport, null otherwise */ export function extractMoqPeerSettings(yamlContent: string): MoqPeerSettings | null { - try { - const parsed = load(yamlContent) as Record | null; - if (!parsed || typeof parsed !== 'object') return null; - - const client = parsed.client as ClientSection | undefined; - if (!client) return null; + const client = parseClientFromYaml(yamlContent); + if (!client) return null; - // Only return settings for dynamic pipelines that declare MoQ transport. - if (!client.gateway_path && !client.relay_url && !client.publish && !client.watch) { - return null; - } - - return deriveSettingsFromClient(client); - } catch { + // Only return settings for dynamic pipelines that declare MoQ transport. + if (!client.gateway_path && !client.relay_url && !client.publish && !client.watch) { return null; } + + return deriveSettingsFromClient(client); } /** diff --git a/ui/src/views/ConvertView.tsx b/ui/src/views/ConvertView.tsx index e4e208c2..714198be 100644 --- a/ui/src/views/ConvertView.tsx +++ b/ui/src/views/ConvertView.tsx @@ -39,12 +39,12 @@ import { } from '@/services/converter'; import { listSamples } from '@/services/samples'; import { ensureSchemasLoaded, useSchemaStore } from '@/stores/schemaStore'; +import { parseAcceptToFormats, parseClientFromYaml } from '@/utils/clientSection'; import { viewsLogger } from '@/utils/logger'; import { orderSamplePipelinesSystemFirst } from '@/utils/samplePipelineOrdering'; import { injectFileReadNode } from '@/utils/yamlPipeline'; type HttpInputField = { name: string; required: boolean }; -type InputFormatSpec = { all: string[]; perField: Record }; const resolveUploadFields = (httpInputFields: HttpInputField[]): HttpInputField[] => httpInputFields.length > 0 ? httpInputFields : [{ name: 'media', required: true }]; @@ -360,91 +360,6 @@ const CharCounter = styled.div` text-align: right; `; -// Helper functions moved outside component (pure functions, no dependencies) - -/** - * Detects if the current pipeline is a transcription pipeline - */ -const checkIfTranscriptionPipeline = (yaml: string): boolean => { - // A transcription pipeline is one that produces `Transcription` packets. - // `core::json_serialize` is used by many pipelines (VAD events, etc.) so it is not a signal. - const lowerYaml = yaml.toLowerCase(); - return ( - lowerYaml.includes('plugin::native::whisper') || - lowerYaml.includes('plugin::native::sensevoice') || - lowerYaml.includes('transcription') - ); -}; - -/** - * Detects if the current pipeline generates its own input (no user input needed) - */ -const checkIfNoInputPipeline = (yaml: string): boolean => { - const lowerYaml = yaml.toLowerCase(); - - // Check if pipeline starts with a script node that uses fetch() - // This indicates the pipeline generates its own data - if (lowerYaml.includes('core::script') && lowerYaml.includes('fetch')) { - return true; - } - - // Pipelines that have http_output but no http_input are self-contained generators - // (e.g. video::colorbars → encoder → muxer → http_output) - if ( - lowerYaml.includes('streamkit::http_output') && - !lowerYaml.includes('streamkit::http_input') - ) { - return true; - } - - return false; -}; - -/** - * Detects if the current pipeline is a TTS pipeline (text input) - */ -const checkIfTTSPipeline = (yaml: string): boolean => { - // First check if it's a no-input pipeline (takes precedence) - if (checkIfNoInputPipeline(yaml)) { - return false; - } - - // A TTS pipeline for text input should have text_chunker as an early node - // Just having TTS nodes isn't enough - the pipeline might use TTS as a component - // in a larger audio-to-audio pipeline (like speech translation) - const lowerYaml = yaml.toLowerCase(); - - // Check for text_chunker which indicates text input processing - if (lowerYaml.includes('text_chunker')) { - return true; - } - - // Additional heuristic: If we have TTS but NO audio demuxers/decoders, - // it's likely a text input pipeline - const hasTTS = - lowerYaml.includes('kokoro_tts') || - lowerYaml.includes('piper_tts') || - lowerYaml.includes('text-to-speech'); - - const hasAudioDemuxer = lowerYaml.includes('demux') || lowerYaml.includes('decode'); - - // If we have TTS but no audio demuxer, it's a text input pipeline - return hasTTS && !hasAudioDemuxer; -}; - -/** - * Detects if the current pipeline produces video output. - * Checks for `video::` node kind prefixes, and `video_width` / `video_height` - * when they appear as YAML mapping keys (not inside comments or arbitrary strings). - */ -const checkIfVideoPipeline = (yaml: string): boolean => { - const lowerYaml = yaml.toLowerCase(); - if (lowerYaml.includes('video::')) return true; - // Match video_width / video_height only as YAML keys (leading whitespace + colon suffix) - // to avoid false-positives on comments or unrelated string values. - return /^\s*video_width\s*:/m.test(lowerYaml) || /^\s*video_height\s*:/m.test(lowerYaml); -}; - const resolveTextField = (fields: HttpInputField[]): HttpInputField | null => { if (fields.length === 0) { return null; @@ -466,77 +381,6 @@ const splitTtsFields = ( }; }; -const FORMAT_ACCEPT_MAP: Record = { - ogg: '.ogg', - opus: '.opus', - mp3: '.mp3', - wav: '.wav', - flac: '.flac', - txt: '.txt', - text: '.txt', - json: '.json', -}; - -const detectInputFormatSpec = (yaml: string): InputFormatSpec | null => { - const match = yaml.match(/^\s*#\s*skit:input_formats\s*=\s*([^\n#]+)\s*$/im); - if (!match?.[1]) return null; - - const spec: InputFormatSpec = { all: [], perField: {} }; - - for (const entry of match[1].split(',')) { - const token = entry.trim(); - if (!token) continue; - - const parts = token.split(':'); - if (parts.length > 1) { - const field = parts[0].trim().toLowerCase(); - const formats = parts - .slice(1) - .join(':') - .split(/[|+]/) - .map((format) => format.trim().toLowerCase()) - .filter(Boolean); - - if (!field || formats.length === 0) continue; - - const existing = spec.perField[field] ?? []; - for (const format of formats) { - if (!existing.includes(format)) { - existing.push(format); - } - } - spec.perField[field] = existing; - } else { - const format = token.toLowerCase(); - if (!spec.all.includes(format)) { - spec.all.push(format); - } - } - } - - if (spec.all.length === 0 && Object.keys(spec.perField).length === 0) { - return null; - } - - return spec; -}; - -const resolveFormatsForField = ( - fieldName: string, - formatSpec: InputFormatSpec | null, - fallbackFormats: string[] | null -): string[] | null => { - const key = fieldName.toLowerCase(); - const perField = formatSpec?.perField?.[key]; - if (perField && perField.length > 0) { - return perField; - } - if (formatSpec?.all && formatSpec.all.length > 0) { - return formatSpec.all; - } - return fallbackFormats; -}; - const buildNoInputUploads = (fields: HttpInputField[]): UploadField[] => { // Generator pipelines (e.g. video::colorbars) have no http_input at all — // return empty so the multipart request only contains the config field. @@ -603,37 +447,12 @@ const buildUploadModeUploads = ( return [{ field: fields[0].name, file: selectedFile }]; }; -const formatHintForField = ( - field: HttpInputField, - formatSpec: InputFormatSpec | null, - fallbackFormats: string[] | null, - isTts: boolean -): { accept?: string; hint?: string } => { - const name = field.name.toLowerCase(); - const fieldOverrides = - (formatSpec?.all?.length ?? 0) > 0 || (formatSpec?.perField?.[name]?.length ?? 0) > 0; - - if (isTts && name.includes('voice') && !fieldOverrides) { - return { accept: 'audio/wav,.wav,.wave', hint: 'Expected format: WAV audio' }; - } - - const formats = resolveFormatsForField(field.name, formatSpec, fallbackFormats); - if (!formats || formats.length === 0) { - return {}; - } - - const unique = Array.from(new Set(formats.map((format) => format.toLowerCase()))); - const accept = unique - .map((format) => FORMAT_ACCEPT_MAP[format]) - .filter(Boolean) - .join(','); - const label = unique.map((format) => format.toUpperCase()).join(', '); - const hint = `Expected format${unique.length > 1 ? 's' : ''}: ${label}`; - - return { - accept: accept || undefined, - hint, - }; +const assetMatchesTag = (assetId: string, tag: string): boolean => { + if (tag === 'speech') return assetId.toLowerCase().startsWith('speech_'); + if (tag === 'music') return assetId.toLowerCase().startsWith('music_'); + if (tag.startsWith('id:')) + return assetId.toLowerCase() === tag.slice('id:'.length).trim().toLowerCase(); + return false; }; /** @@ -731,12 +550,6 @@ const ConvertView: React.FC = () => { setPipelineYaml, selectedTemplateId, setSelectedTemplateId, - isTranscriptionPipeline, - setIsTranscriptionPipeline, - isTTSPipeline, - setIsTTSPipeline, - isNoInputPipeline, - setIsNoInputPipeline, textInput, setTextInput, conversionStatus, @@ -769,8 +582,12 @@ const ConvertView: React.FC = () => { const [msePlaybackError, setMsePlaybackError] = useState(null); const [mseFallbackLoading, setMseFallbackLoading] = useState(false); - // Derived: detect if the selected pipeline produces video output - const isVideoPipeline = useMemo(() => checkIfVideoPipeline(pipelineYaml), [pipelineYaml]); + // Derive pipeline characteristics from declarative client section + const client = useMemo(() => parseClientFromYaml(pipelineYaml), [pipelineYaml]); + const isTranscriptionPipeline = client?.output?.type === 'transcription'; + const isTTSPipeline = client?.input?.type === 'text'; + const isNoInputPipeline = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + const isVideoPipeline = client?.output?.type === 'video'; // Generate CLI command based on current template and pipeline type const cliCommand = useMemo(() => { @@ -823,101 +640,39 @@ const ConvertView: React.FC = () => { // Fetch audio assets const { data: audioAssets = [], isLoading: assetsLoading } = useAudioAssets(); - /** - * Detects the expected input format(s) from a pipeline YAML - * Returns an array of compatible formats, or null if any format is acceptable - */ - const detectExpectedFormats = (yaml: string): string[] | null => { - const lowerYaml = yaml.toLowerCase(); - - // If there's no demuxer/decoder node, any format might work (e.g., passthrough pipelines) - const hasDecoder = lowerYaml.includes('demux') || lowerYaml.includes('decode'); - if (!hasDecoder) { - return null; // Accept all formats - } - - const compatibleFormats: string[] = []; - - // OGG container (opus, vorbis) - // Match patterns: ogg::demuxer, ogg_demux, opus::decoder, opus_decode - if ( - lowerYaml.includes('ogg::demux') || - lowerYaml.includes('ogg_demux') || - lowerYaml.includes('opus::decode') || - lowerYaml.includes('opus_decode') - ) { - compatibleFormats.push('ogg', 'opus'); - } - - // FLAC - if (lowerYaml.includes('flac')) { - compatibleFormats.push('flac'); - } - - // WAV/PCM - if (lowerYaml.includes('wav') || lowerYaml.includes('pcm')) { - compatibleFormats.push('wav'); - } - - // MP3 - if (lowerYaml.includes('mp3')) { - compatibleFormats.push('mp3'); - } - - // If we found specific formats, return them; otherwise return null (accept all) - return compatibleFormats.length > 0 ? compatibleFormats : null; - }; - - /** - * Detects optional asset tags for Convert view's asset picker. - * - * This is a UI-only hint, carried in YAML comments so it doesn't affect pipeline parsing. - * - * Format: - * # skit:input_asset_tags=speech,music - */ - const detectInputAssetTags = (yaml: string): string[] | null => { - const match = yaml.match(/^\s*#\s*skit:input_asset_tags\s*=\s*([^\n#]+)\s*$/im); - if (!match?.[1]) return null; - - const tags = match[1] - .split(',') - .map((tag) => tag.trim().toLowerCase()) - .filter(Boolean); - - return tags.length > 0 ? tags : null; - }; - - const assetMatchesTag = (assetId: string, tag: string): boolean => { - if (tag === 'speech') { - return assetId.toLowerCase().startsWith('speech_'); - } - - if (tag === 'music') { - return assetId.toLowerCase().startsWith('music_'); - } - - if (tag.startsWith('id:')) { - return assetId.toLowerCase() === tag.slice('id:'.length).trim().toLowerCase(); - } - - return false; - }; - - // Filter assets based on pipeline's expected format; for multi-field uploads, allow all assets so fields can mix - const inputFormatSpec = React.useMemo(() => detectInputFormatSpec(pipelineYaml), [pipelineYaml]); - const inferredFormats = React.useMemo(() => detectExpectedFormats(pipelineYaml), [pipelineYaml]); - const assetFieldName = httpInputFields.length > 0 ? httpInputFields[0].name : 'media'; - const assetFormats = React.useMemo( - () => resolveFormatsForField(assetFieldName, inputFormatSpec, inferredFormats), - [assetFieldName, inputFormatSpec, inferredFormats] + // Derive a field-hint lookup from the client section + const getFieldHint = useCallback( + (fieldName: string): { accept?: string; hint?: string } => { + const hints = client?.input?.field_hints; + if (hints) { + const fh = hints[fieldName]; + if (fh) { + const formats = parseAcceptToFormats(fh.accept); + if (formats && formats.length > 0) { + const accept = formats.map((f) => `.${f}`).join(','); + const label = formats.map((f) => f.toUpperCase()).join(', '); + return { accept, hint: `Expected format${formats.length > 1 ? 's' : ''}: ${label}` }; + } + if (fh.accept) return { accept: fh.accept }; + } + } + // Fall back to top-level input accept + const topFormats = parseAcceptToFormats(client?.input?.accept); + if (topFormats && topFormats.length > 0) { + const accept = topFormats.map((f) => `.${f}`).join(','); + const label = topFormats.map((f) => f.toUpperCase()).join(', '); + return { accept, hint: `Expected format${topFormats.length > 1 ? 's' : ''}: ${label}` }; + } + return {}; + }, + [client] ); - const filteredAssets = React.useMemo(() => { - if (!pipelineYaml) { - return audioAssets; - } - const inputAssetTags = detectInputAssetTags(pipelineYaml); + // Filter assets based on client-declared accept & asset_tags + const assetFormats = useMemo(() => parseAcceptToFormats(client?.input?.accept), [client]); + const inputAssetTags = client?.input?.asset_tags ?? null; + const filteredAssets = useMemo(() => { + if (!pipelineYaml) return audioAssets; // Multi-field pipelines: only filter by format (avoid tag-based narrowing so users can mix content) if (httpInputFields.length > 1) { @@ -933,7 +688,6 @@ const ConvertView: React.FC = () => { viewsLogger.debug('Expected formats:', assetFormats, 'Total assets:', audioAssets.length); - // Filter assets to only those with compatible formats const formatFiltered = assetFormats ? audioAssets.filter((asset) => assetFormats.includes(asset.format.toLowerCase())) : audioAssets; @@ -947,7 +701,7 @@ const ConvertView: React.FC = () => { viewsLogger.debug('Filtered to', tagFiltered.length, 'compatible assets'); return tagFiltered; - }, [audioAssets, pipelineYaml, httpInputFields.length, assetFormats]); + }, [audioAssets, pipelineYaml, httpInputFields.length, assetFormats, inputAssetTags]); // Clear selected asset if it's no longer in the filtered list useEffect(() => { @@ -971,30 +725,12 @@ const ConvertView: React.FC = () => { }); }, [pipelineYaml]); - // Watch for pipeline YAML changes and update transcription/TTS detection + // Force playback mode for transcription/TTS pipelines useEffect(() => { - const isTranscription = checkIfTranscriptionPipeline(pipelineYaml); - const isTTS = checkIfTTSPipeline(pipelineYaml); - const isNoInput = checkIfNoInputPipeline(pipelineYaml); - setIsTranscriptionPipeline(isTranscription); - setIsTTSPipeline(isTTS); - setIsNoInputPipeline(isNoInput); - // Force playback mode for transcription pipelines - if (isTranscription && outputMode !== 'playback') { - setOutputMode('playback'); - } - // TTS pipelines always output audio, so default to playback - if (isTTS && outputMode !== 'playback') { + if ((isTranscriptionPipeline || isTTSPipeline) && outputMode !== 'playback') { setOutputMode('playback'); } - }, [ - pipelineYaml, - outputMode, - setIsTranscriptionPipeline, - setIsTTSPipeline, - setIsNoInputPipeline, - setOutputMode, - ]); + }, [isTranscriptionPipeline, isTTSPipeline, outputMode, setOutputMode]); // Update YAML when asset selection changes useEffect(() => { @@ -1038,7 +774,7 @@ const ConvertView: React.FC = () => { const defaultSample = orderedSamples[0]; setSelectedTemplateId(defaultSample.id); setPipelineYaml(defaultSample.yaml); - setIsTranscriptionPipeline(checkIfTranscriptionPipeline(defaultSample.yaml)); + // Client-section flags are derived via useMemo, no manual setter needed } } catch (error) { viewsLogger.error('Failed to fetch samples:', error); @@ -1049,14 +785,7 @@ const ConvertView: React.FC = () => { }; fetchSamples(); - }, [ - setSamples, - setSamplesLoading, - setSamplesError, - setSelectedTemplateId, - setPipelineYaml, - setIsTranscriptionPipeline, - ]); + }, [setSamples, setSamplesLoading, setSamplesError, setSelectedTemplateId, setPipelineYaml]); const handleTemplateSelect = (templateId: string) => { const sample = samples.find((s) => s.id === templateId); @@ -1068,9 +797,10 @@ const ConvertView: React.FC = () => { // Set original YAML (asset selection will be reapplied via useEffect if needed) setPipelineYaml(sample.yaml); - setIsTranscriptionPipeline(checkIfTranscriptionPipeline(sample.yaml)); - // Force playback mode for transcription pipelines - if (checkIfTranscriptionPipeline(sample.yaml)) { + // Force playback for transcription pipelines (will be picked up by the + // effect once pipelineYaml updates and client is re-derived). + const templateClient = parseClientFromYaml(sample.yaml); + if (templateClient?.output?.type === 'transcription') { setOutputMode('playback'); } } @@ -1426,10 +1156,7 @@ const ConvertView: React.FC = () => { const ttsMissingRequiredUploads = ttsExtraFields.some( (field) => field.required && !fieldUploads[field.name] ); - const singleUploadHint = - uploadFields.length > 0 - ? formatHintForField(uploadFields[0], inputFormatSpec, inferredFormats, isTTSPipeline) - : {}; + const singleUploadHint = uploadFields.length > 0 ? getFieldHint(uploadFields[0].name) : {}; const handleDownloadAudio = () => { if (!mediaUrl) return; @@ -1604,12 +1331,7 @@ const ConvertView: React.FC = () => {
Additional uploads: {ttsExtraFields.map((field) => { - const hint = formatHintForField( - field, - inputFormatSpec, - inferredFormats, - isTTSPipeline - ); + const hint = getFieldHint(field.name); return (
@@ -1658,12 +1380,7 @@ const ConvertView: React.FC = () => { or pick an existing asset.

{uploadFields.map((field) => { - const hint = formatHintForField( - field, - inputFormatSpec, - inferredFormats, - isTTSPipeline - ); + const hint = getFieldHint(field.name); return (
From af28675d5b173f9ea945e2cb79b46c92ed848da2 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:27:00 +0000 Subject: [PATCH 05/20] feat(api): add lint_client_section() with 12 semantic validation rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a lint pass that validates the client section against the pipeline mode and internal consistency. Returns warnings (never blocks compilation). Rules: 1. mode-mismatch-dynamic — dynamic with oneshot fields 2. mode-mismatch-oneshot — oneshot with dynamic fields 3. missing-gateway — publish/watch without gateway_path or relay_url 4. publish-no-media — publish with both audio+video false 5. watch-no-media — watch with both audio+video false 6. input-none-with-accept — none input with accept set 7. input-trigger-with-accept — trigger input with accept set 8. field-hints-no-input — field_hints on none input 9. asset-tags-no-input — asset_tags on none/text input 10. text-no-placeholder — text input missing placeholder 11. empty-broadcast — empty broadcast string 12. duplicate-broadcast — publish.broadcast == watch.broadcast Includes 14 lint tests covering all rules plus clean-pass cases. Co-Authored-By: Claudio Costa --- crates/api/src/yaml.rs | 423 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 423 insertions(+) diff --git a/crates/api/src/yaml.rs b/crates/api/src/yaml.rs index 04ced9b6..51190995 100644 --- a/crates/api/src/yaml.rs +++ b/crates/api/src/yaml.rs @@ -568,6 +568,209 @@ fn compile_dag( Ok(Pipeline { name, description, mode, client, nodes, connections, view_data: None }) } +// --------------------------------------------------------------------------- +// Client section lint pass — semantic validation +// --------------------------------------------------------------------------- + +/// A single lint warning produced by [`lint_client_section`]. +#[derive(Debug, Clone)] +pub struct ClientLintWarning { + /// Machine-readable rule identifier (e.g. `"mode-mismatch"`). + pub rule: &'static str, + /// Human-readable description of the problem. + pub message: String, +} + +/// Validates the `client` section against the compiled pipeline, returning +/// any semantic warnings. +/// +/// This is a *lint* pass — it never prevents compilation, but surfaces +/// likely authoring mistakes so tooling (CLI, editor integrations) can +/// flag them. +/// +/// # Rules +/// +/// 1. **`mode-mismatch-dynamic`** — Dynamic pipeline declares oneshot-only +/// fields (`input` / `output`). +/// 2. **`mode-mismatch-oneshot`** — Oneshot pipeline declares dynamic-only +/// fields (`publish` / `watch` / `gateway_path` / `relay_url`). +/// 3. **`missing-gateway`** — Dynamic pipeline has `publish` or `watch` +/// but no `gateway_path` or `relay_url`. +/// 4. **`publish-no-media`** — `publish` block sets both `audio` and +/// `video` to false. +/// 5. **`watch-no-media`** — `watch` block sets both `audio` and `video` +/// to false. +/// 6. **`input-none-with-accept`** — `input.type` is `none` but `accept` +/// is set (accept is meaningless without a file picker). +/// 7. **`input-trigger-with-accept`** — `input.type` is `trigger` but +/// `accept` is set. +/// 8. **`field-hints-no-input`** — `field_hints` is present but +/// `input.type` is `none`. +/// 9. **`asset-tags-no-input`** — `asset_tags` is present but +/// `input.type` is `none` or `text`. +/// 10. **`text-no-placeholder`** — `input.type` is `text` but no +/// `placeholder` is provided (best-practice hint). +/// 11. **`empty-broadcast`** — `publish.broadcast` or `watch.broadcast` +/// is an empty string. +/// 12. **`duplicate-broadcast`** — `publish.broadcast` equals +/// `watch.broadcast` (would cause a loop). +pub fn lint_client_section(client: &ClientSection, mode: EngineMode) -> Vec { + let mut warnings = Vec::new(); + + let has_dynamic_fields = client.gateway_path.is_some() + || client.relay_url.is_some() + || client.publish.is_some() + || client.watch.is_some(); + + let has_oneshot_fields = client.input.is_some() || client.output.is_some(); + + // Rule 1: dynamic pipeline with oneshot-only fields + if mode == EngineMode::Dynamic && has_oneshot_fields { + warnings.push(ClientLintWarning { + rule: "mode-mismatch-dynamic", + message: "Dynamic pipeline declares `input` or `output` — these are oneshot-only \ + fields and will be ignored." + .into(), + }); + } + + // Rule 2: oneshot pipeline with dynamic-only fields + if mode == EngineMode::OneShot && has_dynamic_fields { + warnings.push(ClientLintWarning { + rule: "mode-mismatch-oneshot", + message: "Oneshot pipeline declares `publish`, `watch`, `gateway_path`, or \ + `relay_url` — these are dynamic-only fields and will be ignored." + .into(), + }); + } + + // Rule 3: missing gateway + if (client.publish.is_some() || client.watch.is_some()) + && client.gateway_path.is_none() + && client.relay_url.is_none() + { + warnings.push(ClientLintWarning { + rule: "missing-gateway", + message: "Pipeline has `publish` or `watch` but no `gateway_path` or `relay_url` — \ + the browser won't know where to connect." + .into(), + }); + } + + // Rule 4: publish with no media + if let Some(ref publish) = client.publish { + if !publish.audio && !publish.video { + warnings.push(ClientLintWarning { + rule: "publish-no-media", + message: "publish block sets both `audio` and `video` to false — nothing will be \ + sent from the browser." + .into(), + }); + } + + // Rule 11a: empty broadcast + if publish.broadcast.is_empty() { + warnings.push(ClientLintWarning { + rule: "empty-broadcast", + message: "publish.broadcast is an empty string.".into(), + }); + } + } + + // Rule 5: watch with no media + if let Some(ref watch) = client.watch { + if !watch.audio && !watch.video { + warnings.push(ClientLintWarning { + rule: "watch-no-media", + message: "watch block sets both `audio` and `video` to false — nothing will be \ + received by the browser." + .into(), + }); + } + + // Rule 11b: empty broadcast + if watch.broadcast.is_empty() { + warnings.push(ClientLintWarning { + rule: "empty-broadcast", + message: "watch.broadcast is an empty string.".into(), + }); + } + } + + // Rule 12: duplicate broadcast + if let (Some(ref publish), Some(ref watch)) = (&client.publish, &client.watch) { + if !publish.broadcast.is_empty() && publish.broadcast == watch.broadcast { + warnings.push(ClientLintWarning { + rule: "duplicate-broadcast", + message: format!( + "publish.broadcast and watch.broadcast are both '{}' — this would \ + cause a feedback loop.", + publish.broadcast + ), + }); + } + } + + // Input-related rules + if let Some(ref input) = client.input { + // Rule 6: input none with accept + if matches!(input.input_type, InputType::None) && input.accept.is_some() { + warnings.push(ClientLintWarning { + rule: "input-none-with-accept", + message: "input.type is `none` but `accept` is set — accept is meaningless \ + without a file picker." + .into(), + }); + } + + // Rule 7: input trigger with accept + if matches!(input.input_type, InputType::Trigger) && input.accept.is_some() { + warnings.push(ClientLintWarning { + rule: "input-trigger-with-accept", + message: "input.type is `trigger` but `accept` is set — accept is meaningless \ + for trigger inputs." + .into(), + }); + } + + // Rule 8: field_hints with no input + if matches!(input.input_type, InputType::None) + && input.field_hints.as_ref().is_some_and(|h| !h.is_empty()) + { + warnings.push(ClientLintWarning { + rule: "field-hints-no-input", + message: "field_hints is present but input.type is `none` — hints are unused \ + without an input." + .into(), + }); + } + + // Rule 9: asset_tags with no input or text input + if matches!(input.input_type, InputType::None | InputType::Text) + && input.asset_tags.as_ref().is_some_and(|t| !t.is_empty()) + { + warnings.push(ClientLintWarning { + rule: "asset-tags-no-input", + message: "asset_tags is present but input.type is `none` or `text` — tags are \ + only useful for file_upload inputs." + .into(), + }); + } + + // Rule 10: text input without placeholder + if matches!(input.input_type, InputType::Text) && input.placeholder.is_none() { + warnings.push(ClientLintWarning { + rule: "text-no-placeholder", + message: "input.type is `text` but no `placeholder` is provided — consider \ + adding one for a better UX." + .into(), + }); + } + } + + warnings +} + #[cfg(test)] mod tests { use super::*; @@ -1294,4 +1497,224 @@ client: let tags = input.asset_tags.expect("asset_tags should be present"); assert_eq!(tags, vec!["speech", "voice"]); } + + // ----------------------------------------------------------------------- + // Client section lint tests + // ----------------------------------------------------------------------- + + /// Helper to build a minimal valid dynamic client section. + fn dynamic_client() -> ClientSection { + ClientSection { + relay_url: None, + gateway_path: Some("/moq/test".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + watch: Some(WatchConfig { broadcast: "output".into(), audio: true, video: true }), + input: None, + output: None, + } + } + + /// Helper to build a minimal valid oneshot client section. + fn oneshot_client() -> ClientSection { + ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + } + } + + #[test] + fn test_lint_clean_dynamic() { + let warnings = lint_client_section(&dynamic_client(), EngineMode::Dynamic); + assert!(warnings.is_empty(), "Expected no warnings: {warnings:?}"); + } + + #[test] + fn test_lint_clean_oneshot() { + let warnings = lint_client_section(&oneshot_client(), EngineMode::OneShot); + assert!(warnings.is_empty(), "Expected no warnings: {warnings:?}"); + } + + #[test] + fn test_lint_mode_mismatch_dynamic_with_oneshot_fields() { + let mut c = dynamic_client(); + c.input = Some(InputConfig { + input_type: InputType::FileUpload, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: None, + }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "mode-mismatch-dynamic")); + } + + #[test] + fn test_lint_mode_mismatch_oneshot_with_dynamic_fields() { + let mut c = oneshot_client(); + c.gateway_path = Some("/moq/test".into()); + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "mode-mismatch-oneshot")); + } + + #[test] + fn test_lint_missing_gateway() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: Some(PublishConfig { broadcast: "x".into(), audio: true, video: false }), + watch: None, + input: None, + output: None, + }; + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "missing-gateway")); + } + + #[test] + fn test_lint_publish_no_media() { + let mut c = dynamic_client(); + c.publish = Some(PublishConfig { broadcast: "x".into(), audio: false, video: false }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "publish-no-media")); + } + + #[test] + fn test_lint_watch_no_media() { + let mut c = dynamic_client(); + c.watch = Some(WatchConfig { broadcast: "x".into(), audio: false, video: false }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "watch-no-media")); + } + + #[test] + fn test_lint_empty_broadcast() { + let mut c = dynamic_client(); + c.publish = Some(PublishConfig { broadcast: String::new(), audio: true, video: false }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "empty-broadcast")); + } + + #[test] + fn test_lint_duplicate_broadcast() { + let mut c = dynamic_client(); + c.publish = Some(PublishConfig { broadcast: "same".into(), audio: true, video: false }); + c.watch = Some(WatchConfig { broadcast: "same".into(), audio: true, video: true }); + let warnings = lint_client_section(&c, EngineMode::Dynamic); + assert!(warnings.iter().any(|w| w.rule == "duplicate-broadcast")); + } + + #[test] + fn test_lint_input_none_with_accept() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::None, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Video }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "input-none-with-accept")); + } + + #[test] + fn test_lint_input_trigger_with_accept() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::Trigger, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "input-trigger-with-accept")); + } + + #[test] + fn test_lint_field_hints_no_input() { + let mut hints = IndexMap::new(); + hints.insert( + "x".into(), + FieldHint { field_type: Some(FieldType::File), accept: None, placeholder: None }, + ); + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::None, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Video }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "field-hints-no-input")); + } + + #[test] + fn test_lint_asset_tags_text_input() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::Text, + accept: None, + asset_tags: Some(vec!["speech".into()]), + placeholder: Some("Enter text".into()), + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "asset-tags-no-input")); + } + + #[test] + fn test_lint_text_no_placeholder() { + let c = ClientSection { + relay_url: None, + gateway_path: None, + publish: None, + watch: None, + input: Some(InputConfig { + input_type: InputType::Text, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + }; + let warnings = lint_client_section(&c, EngineMode::OneShot); + assert!(warnings.iter().any(|w| w.rule == "text-no-placeholder")); + } } From f052cf73cdf1b7a120996e2859c1e3241c4e5668 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:29:23 +0000 Subject: [PATCH 06/20] fix: break circular import between clientSection and moqPeerSettings Remove the import of MoqPeerSettings type from clientSection.ts to prevent a circular module dependency that caused CI test failures. Inline the return type shape instead. Co-Authored-By: Claudio Costa --- ui/src/utils/clientSection.ts | 63 +++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/ui/src/utils/clientSection.ts b/ui/src/utils/clientSection.ts index 1c6b38ee..f577aab3 100644 --- a/ui/src/utils/clientSection.ts +++ b/ui/src/utils/clientSection.ts @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: MPL-2.0 -import type { ClientSection, Pipeline } from '@/types/types'; +import { load } from 'js-yaml'; -import type { MoqPeerSettings } from './moqPeerSettings'; +import type { ClientSection, Pipeline } from '@/types/types'; /** * Extracts the `client` section from a compiled `Pipeline`, returning `null` @@ -15,13 +15,62 @@ export function extractClientSection(pipeline: Pipeline | null | undefined): Cli } /** - * Derives `MoqPeerSettings` from a declarative `ClientSection`. + * Parses a raw pipeline YAML string and extracts the `client` section. + * Returns `null` if the YAML is invalid or has no client section. + */ +export function parseClientFromYaml(yamlContent: string): ClientSection | null { + try { + const parsed = load(yamlContent) as Record | null; + if (!parsed || typeof parsed !== 'object') return null; + return (parsed.client as ClientSection) ?? null; + } catch { + return null; + } +} + +/** + * Converts a CSS-style `accept` attribute (e.g. `"audio/*"`, `".ogg,.opus"`) + * into an array of lowercase format names suitable for asset-format matching. + * Returns `null` when all formats are acceptable. + */ +export function parseAcceptToFormats(accept: string | null | undefined): string[] | null { + if (!accept) return null; + if (accept.includes('*')) return null; + + const formats: string[] = []; + for (const part of accept.split(',')) { + const trimmed = part.trim(); + if (!trimmed) continue; + if (trimmed.startsWith('.')) { + formats.push(trimmed.slice(1).toLowerCase()); + } else if (trimmed.includes('/')) { + const sub = trimmed.split('/')[1]?.toLowerCase(); + if (sub) formats.push(sub); + } else { + formats.push(trimmed.toLowerCase()); + } + } + return formats.length > 0 ? formats : null; +} + +/** + * Derives `MoqPeerSettings`-shaped data from a declarative `ClientSection`. * - * This is the client-section counterpart of `extractMoqPeerSettings()` (which - * heuristically inspects raw YAML). When a pipeline carries a `client` - * section the UI should prefer this function. + * The return type mirrors the `MoqPeerSettings` interface defined in + * `moqPeerSettings.ts`. We avoid importing that interface here to prevent + * a circular module dependency. */ -export function deriveSettingsFromClient(client: ClientSection): MoqPeerSettings { +export function deriveSettingsFromClient(client: ClientSection): { + gatewayPath?: string; + relayUrl?: string; + inputBroadcast?: string; + outputBroadcast?: string; + hasInputBroadcast: boolean; + needsAudioInput: boolean; + needsVideoInput: boolean; + outputsAudio: boolean; + outputsVideo: boolean; +} { return { gatewayPath: client.gateway_path ?? undefined, relayUrl: client.relay_url ?? undefined, From b01003d473d8c86b04ea2fd8f1a4587383e07090 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:35:04 +0000 Subject: [PATCH 07/20] fix: resolve clippy warnings (large_enum_variant, assigning_clones) - Box ApiPipeline in ResponsePayload::Pipeline to fix large_enum_variant - Use clone_from() in populate_session_pipeline for assigning_clones - Add missing client field in compositor benchmark Co-Authored-By: Claudio Costa --- apps/skit/src/server.rs | 6 +++--- apps/skit/src/websocket_handlers.rs | 2 +- crates/api/src/lib.rs | 2 +- crates/engine/benches/compositor_pipeline.rs | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/skit/src/server.rs b/apps/skit/src/server.rs index 68228cf5..987a493b 100644 --- a/apps/skit/src/server.rs +++ b/apps/skit/src/server.rs @@ -1521,10 +1521,10 @@ async fn populate_session_pipeline(session: &crate::session::Session, engine_pip let mut pipeline = session.pipeline.lock().await; // Forward top-level metadata so the UI can read it from the session snapshot. - pipeline.name = engine_pipeline.name.clone(); - pipeline.description = engine_pipeline.description.clone(); + pipeline.name.clone_from(&engine_pipeline.name); + pipeline.description.clone_from(&engine_pipeline.description); pipeline.mode = engine_pipeline.mode; - pipeline.client = engine_pipeline.client.clone(); + pipeline.client.clone_from(&engine_pipeline.client); // Add nodes to in-memory pipeline for (node_id, node_spec) in &engine_pipeline.nodes { diff --git a/apps/skit/src/websocket_handlers.rs b/apps/skit/src/websocket_handlers.rs index 96cef940..248a448b 100644 --- a/apps/skit/src/websocket_handlers.rs +++ b/apps/skit/src/websocket_handlers.rs @@ -1105,7 +1105,7 @@ async fn handle_get_pipeline( "Retrieved pipeline" ); - Some(ResponsePayload::Pipeline { pipeline: api_pipeline }) + Some(ResponsePayload::Pipeline { pipeline: Box::new(api_pipeline) }) } fn handle_validate_batch( diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index b72706ce..fd72147c 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -328,7 +328,7 @@ pub enum ResponsePayload { nodes: Vec, }, Pipeline { - pipeline: ApiPipeline, + pipeline: Box, }, ValidationResult { errors: Vec, diff --git a/crates/engine/benches/compositor_pipeline.rs b/crates/engine/benches/compositor_pipeline.rs index bdd3789c..5c2347db 100644 --- a/crates/engine/benches/compositor_pipeline.rs +++ b/crates/engine/benches/compositor_pipeline.rs @@ -254,6 +254,7 @@ fn build_pipeline(width: u32, height: u32, fps: u32, frame_count: u32) -> stream nodes, connections, view_data: None, + client: None, } } From e0d9d93d6d7dfd4c205f7331ffa65842233114d9 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:36:20 +0000 Subject: [PATCH 08/20] fix: correct pocket-tts-voice-clone input type and no-input fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Change pocket-tts-voice-clone.yml input.type from file_upload to text (matches other TTS pipelines — the old heuristic detected text_chunker) - Add fallback in prepareUploads for custom pipelines without a client section that also lack http_input, preventing silent no-ops Co-Authored-By: Claudio Costa --- samples/pipelines/oneshot/pocket-tts-voice-clone.yml | 2 +- ui/src/views/ConvertView.tsx | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/samples/pipelines/oneshot/pocket-tts-voice-clone.yml b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml index 10611131..5bbbcf38 100644 --- a/samples/pipelines/oneshot/pocket-tts-voice-clone.yml +++ b/samples/pipelines/oneshot/pocket-tts-voice-clone.yml @@ -7,7 +7,7 @@ description: Synthesizes speech from text using Pocket TTS with a voice prompt W mode: oneshot client: input: - type: file_upload + type: text field_hints: voice: type: file diff --git a/ui/src/views/ConvertView.tsx b/ui/src/views/ConvertView.tsx index 714198be..bed95356 100644 --- a/ui/src/views/ConvertView.tsx +++ b/ui/src/views/ConvertView.tsx @@ -809,7 +809,8 @@ const ConvertView: React.FC = () => { const prepareUploads = useCallback(async (): Promise => { // For no-input (generator) pipelines, use the raw httpInputFields // so that truly input-less pipelines (no http_input node) send no uploads. - if (isNoInputPipeline) { + // Also handle custom pipelines without a client section that lack http_input. + if (isNoInputPipeline || (!client && !hasHttpInput)) { return buildNoInputUploads(httpInputFields); } @@ -828,6 +829,7 @@ const ConvertView: React.FC = () => { } return []; }, [ + client, fieldUploads, httpInputFields, inputMode, From 0eb18f14b0f72ebc7ec4490e48efe2f3e13bb6e1 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 18:47:30 +0000 Subject: [PATCH 09/20] fix: add #[serde(default)] to Pipeline.client for correct TS optional type Without #[serde(default)], the generated TS type was 'client: ClientSection | null' instead of 'client?: ClientSection | null', causing runtime undefined vs null mismatch when the JSON key is omitted. Co-Authored-By: Claudio Costa --- crates/api/src/lib.rs | 1 + ui/src/types/generated/api-types.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index fd72147c..67285cbf 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -541,6 +541,7 @@ pub struct Pipeline { /// Declarative UI metadata — forwarded unchanged from `UserPipeline`, /// ignored by the engine for execution. #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub client: Option, #[ts(type = "Record")] pub nodes: indexmap::IndexMap, diff --git a/ui/src/types/generated/api-types.ts b/ui/src/types/generated/api-types.ts index c85c5203..75fc5618 100644 --- a/ui/src/types/generated/api-types.ts +++ b/ui/src/types/generated/api-types.ts @@ -368,7 +368,7 @@ export type Pipeline = { name: string | null, description: string | null, mode: * Declarative UI metadata — forwarded unchanged from `UserPipeline`, * ignored by the engine for execution. */ -client: ClientSection | null, nodes: Record, connections: Array, +client?: ClientSection | null, nodes: Record, connections: Array, /** * Resolved per-node view data (e.g., compositor layout). * Only populated in API responses; absent from pipeline definitions. From c85826a0dbb8338dbf6d791933b09c7c819902e7 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:32:08 +0000 Subject: [PATCH 10/20] =?UTF-8?q?fix:=20close=20remaining=20gaps=20?= =?UTF-8?q?=E2=80=94=20node-graph=20lint=20rules,=20loadtest=20cleanup,=20?= =?UTF-8?q?and=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add lint_client_against_nodes() with 8 cross-validation rules (13-20): input-requires-http-input, input-none-has-http-input, field-hint-unknown-field, publish-no-transport, watch-no-transport, gateway-path-mismatch, relay-url-mismatch, broadcast-mismatch - Add NodeInfo struct for lightweight node representation - Derive Default on ClientSection for ergonomic test construction - Add 16 Rust unit tests covering all 8 new lint rules (positive + negative) - Remove leftover # skit: comment from loadtest sample pipeline - Add 14 UI unit tests for monitor preview config derivation and ConvertView pipeline classification (trigger/none, output-type renderer) Co-Authored-By: Claudio Costa --- crates/api/src/yaml.rs | 573 +++++++++++++++++- .../pipelines/oneshot_mixing_no_pacer.yml | 1 - ui/src/utils/clientSection.test.ts | 213 +++++++ 3 files changed, 785 insertions(+), 2 deletions(-) diff --git a/crates/api/src/yaml.rs b/crates/api/src/yaml.rs index 51190995..912398f6 100644 --- a/crates/api/src/yaml.rs +++ b/crates/api/src/yaml.rs @@ -103,7 +103,7 @@ pub enum Needs { /// Dynamic pipelines use `relay_url`/`gateway_path`/`publish`/`watch`; /// oneshot pipelines use `input`/`output`. The two sets are mutually /// exclusive by mode (enforced by the lint pass, not at parse time). -#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[derive(Debug, Clone, Default, Deserialize, Serialize, TS)] #[ts(export)] pub struct ClientSection { /// Direct relay URL for external MoQ relay pattern. @@ -771,6 +771,265 @@ pub fn lint_client_section(client: &ClientSection, mode: EngineMode) -> Vec { + pub kind: &'a str, + pub params: Option<&'a serde_json::Value>, +} + +/// Cross-validates the `client` section against the pipeline's node graph. +/// +/// This is a second lint layer that complements [`lint_client_section`] +/// (which checks `client` in isolation). The rules here require knowledge +/// of which nodes exist and their params. +/// +/// # Rules +/// +/// 13. **`input-requires-http-input`** — `input.type` is `file_upload`, +/// `text`, or `trigger` but no `streamkit::http_input` node exists. +/// 14. **`input-none-has-http-input`** — `input.type` is `none` but an +/// `streamkit::http_input` node exists (should be `trigger`). +/// 15. **`field-hint-unknown-field`** — `field_hints` references a field +/// name not found in any `streamkit::http_input` node's `fields` param. +/// 16. **`publish-no-transport`** — `publish` is declared but no MoQ +/// transport node (`transport::moq::peer` or +/// `transport::moq::subscriber`) exists. +/// 17. **`watch-no-transport`** — `watch` is declared but no MoQ transport +/// node (`transport::moq::peer` or `transport::moq::publisher`) exists. +/// 18. **`gateway-path-mismatch`** — `client.gateway_path` does not match +/// the `gateway_path` param on a `transport::moq::peer` node. +/// 19. **`relay-url-mismatch`** — `client.relay_url` does not match the +/// `url` param on a `transport::moq::publisher` or +/// `transport::moq::subscriber` node. +/// 20. **`broadcast-mismatch`** — `publish.broadcast` or `watch.broadcast` +/// does not match any broadcast name configured on MoQ transport nodes. +pub fn lint_client_against_nodes( + client: &ClientSection, + _mode: EngineMode, + nodes: &[NodeInfo<'_>], +) -> Vec { + let mut warnings = Vec::new(); + + // Collect node kinds and params for efficient lookup. + let has_http_input = nodes.iter().any(|n| n.kind == "streamkit::http_input"); + let has_moq_peer = nodes.iter().any(|n| n.kind == "transport::moq::peer"); + let has_moq_subscriber = nodes.iter().any(|n| n.kind == "transport::moq::subscriber"); + let has_moq_publisher = nodes.iter().any(|n| n.kind == "transport::moq::publisher"); + + // Rule 13: input requires http_input node + if let Some(ref input) = client.input { + let needs_http_input = matches!( + input.input_type, + InputType::FileUpload | InputType::Text | InputType::Trigger + ); + if needs_http_input && !has_http_input { + warnings.push(ClientLintWarning { + rule: "input-requires-http-input", + message: format!( + "input.type is `{}` but no `streamkit::http_input` node exists.", + match input.input_type { + InputType::FileUpload => "file_upload", + InputType::Text => "text", + InputType::Trigger => "trigger", + InputType::None => "none", + } + ), + }); + } + + // Rule 14: input.type is none but http_input exists + if matches!(input.input_type, InputType::None) && has_http_input { + warnings.push(ClientLintWarning { + rule: "input-none-has-http-input", + message: "input.type is `none` but a `streamkit::http_input` node exists — \ + consider using `trigger` instead." + .into(), + }); + } + + // Rule 15: field_hints references unknown field names + if let Some(ref hints) = input.field_hints { + let mut declared_fields: Vec = Vec::new(); + for node in nodes.iter().filter(|n| n.kind == "streamkit::http_input") { + if let Some(params) = node.params { + // Single field: { field: { name: "foo" } } + if let Some(name) = params.get("field").and_then(|f| f.get("name")).and_then(|n| n.as_str()) { + declared_fields.push(name.to_string()); + } + // Multi field: { fields: [{ name: "foo" }, { name: "bar" }] } + if let Some(fields_arr) = params.get("fields").and_then(|f| f.as_array()) { + for f in fields_arr { + if let Some(name) = f.get("name").and_then(|n| n.as_str()) { + declared_fields.push(name.to_string()); + } + } + } + } + // http_input with no params has a single default field named "media" + if (node.params.is_none() + || node.params.is_none_or(|p| { + p.get("field").is_none() && p.get("fields").is_none() + })) + && !declared_fields.contains(&"media".to_string()) + { + declared_fields.push("media".to_string()); + } + } + + if !declared_fields.is_empty() { + for hint_name in hints.keys() { + if !declared_fields.iter().any(|f| f == hint_name) { + warnings.push(ClientLintWarning { + rule: "field-hint-unknown-field", + message: format!( + "field_hints references `{hint_name}` but no `streamkit::http_input` \ + node declares a field with that name. Known fields: {}.", + declared_fields.join(", ") + ), + }); + } + } + } + } + } + + // Rule 16: publish but no MoQ subscriber/peer + // Browser publish = server subscribes → need moq::peer or moq::subscriber + if client.publish.is_some() && !has_moq_peer && !has_moq_subscriber { + warnings.push(ClientLintWarning { + rule: "publish-no-transport", + message: "client declares `publish` but no `transport::moq::peer` or \ + `transport::moq::subscriber` node exists." + .into(), + }); + } + + // Rule 17: watch but no MoQ publisher/peer + // Browser watch = server publishes → need moq::peer or moq::publisher + if client.watch.is_some() && !has_moq_peer && !has_moq_publisher { + warnings.push(ClientLintWarning { + rule: "watch-no-transport", + message: "client declares `watch` but no `transport::moq::peer` or \ + `transport::moq::publisher` node exists." + .into(), + }); + } + + // Rule 18: gateway_path mismatch with moq::peer node + if let Some(ref client_gw) = client.gateway_path { + let peer_gateway_paths: Vec<&str> = nodes + .iter() + .filter(|n| n.kind == "transport::moq::peer") + .filter_map(|n| { + n.params + .and_then(|p| p.get("gateway_path")) + .and_then(|v| v.as_str()) + }) + .collect(); + + if !peer_gateway_paths.is_empty() + && !peer_gateway_paths.iter().any(|gw| gw == client_gw) + { + warnings.push(ClientLintWarning { + rule: "gateway-path-mismatch", + message: format!( + "client.gateway_path is `{client_gw}` but moq::peer node(s) declare: {}.", + peer_gateway_paths.join(", ") + ), + }); + } + } + + // Rule 19: relay_url mismatch with publisher/subscriber nodes + if let Some(ref client_url) = client.relay_url { + let node_urls: Vec<&str> = nodes + .iter() + .filter(|n| { + n.kind == "transport::moq::publisher" + || n.kind == "transport::moq::subscriber" + }) + .filter_map(|n| { + n.params + .and_then(|p| p.get("url")) + .and_then(|v| v.as_str()) + }) + .collect(); + + if !node_urls.is_empty() && !node_urls.iter().any(|u| u == client_url) { + warnings.push(ClientLintWarning { + rule: "relay-url-mismatch", + message: format!( + "client.relay_url is `{client_url}` but transport node(s) declare: {}.", + node_urls.join(", ") + ), + }); + } + } + + // Rule 20: broadcast name mismatch + // Collect all broadcast names from MoQ transport nodes. + let mut node_broadcasts: Vec<&str> = Vec::new(); + for node in nodes { + if let Some(params) = node.params { + match node.kind { + "transport::moq::peer" => { + if let Some(b) = params.get("input_broadcast").and_then(|v| v.as_str()) { + node_broadcasts.push(b); + } + if let Some(b) = params.get("output_broadcast").and_then(|v| v.as_str()) { + node_broadcasts.push(b); + } + } + "transport::moq::publisher" | "transport::moq::subscriber" => { + if let Some(b) = params.get("broadcast").and_then(|v| v.as_str()) { + node_broadcasts.push(b); + } + } + _ => {} + } + } + } + + if !node_broadcasts.is_empty() { + if let Some(ref publish) = client.publish { + if !publish.broadcast.is_empty() + && !node_broadcasts.iter().any(|b| *b == publish.broadcast) + { + warnings.push(ClientLintWarning { + rule: "broadcast-mismatch", + message: format!( + "publish.broadcast is `{}` but no MoQ transport node declares \ + that broadcast name. Node broadcasts: {}.", + publish.broadcast, + node_broadcasts.join(", ") + ), + }); + } + } + if let Some(ref watch) = client.watch { + if !watch.broadcast.is_empty() + && !node_broadcasts.iter().any(|b| *b == watch.broadcast) + { + warnings.push(ClientLintWarning { + rule: "broadcast-mismatch", + message: format!( + "watch.broadcast is `{}` but no MoQ transport node declares \ + that broadcast name. Node broadcasts: {}.", + watch.broadcast, + node_broadcasts.join(", ") + ), + }); + } + } + } + + warnings +} + #[cfg(test)] mod tests { use super::*; @@ -1717,4 +1976,316 @@ client: let warnings = lint_client_section(&c, EngineMode::OneShot); assert!(warnings.iter().any(|w| w.rule == "text-no-placeholder")); } + + // ----------------------------------------------------------------------- + // Client-vs-nodes cross-validation tests (rules 13–20) + // ----------------------------------------------------------------------- + + /// Helper: a `streamkit::http_input` node with no params. + fn http_input_node() -> serde_json::Value { + serde_json::Value::Null // represents "no params object" + } + + fn node<'a>(kind: &'a str, params: Option<&'a serde_json::Value>) -> NodeInfo<'a> { + NodeInfo { kind, params } + } + + // Rule 13 — input-requires-http-input + #[test] + fn test_lint_input_requires_http_input() { + let c = oneshot_client(); // input.type = file_upload + let nodes: Vec> = vec![]; // no http_input + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!(warnings.iter().any(|w| w.rule == "input-requires-http-input")); + } + + #[test] + fn test_lint_input_requires_http_input_clean() { + let c = oneshot_client(); + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "input-requires-http-input"), + "Should not warn when http_input exists: {warnings:?}" + ); + } + + // Rule 14 — input-none-has-http-input + #[test] + fn test_lint_input_none_has_http_input() { + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::None, + accept: None, + asset_tags: None, + placeholder: None, + field_hints: None, + }), + output: Some(OutputConfig { output_type: OutputType::Video }), + ..Default::default() + }; + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!(warnings.iter().any(|w| w.rule == "input-none-has-http-input")); + } + + // Rule 15 — field-hint-unknown-field + #[test] + fn test_lint_field_hint_unknown_field() { + let mut hints = IndexMap::new(); + hints.insert( + "nonexistent".into(), + FieldHint { field_type: Some(FieldType::Text), accept: None, placeholder: None }, + ); + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + ..Default::default() + }; + // http_input with no explicit field/fields → default field is "media" + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "field-hint-unknown-field"), + "Should warn for unknown field hint name: {warnings:?}" + ); + } + + #[test] + fn test_lint_field_hint_known_field_clean() { + let mut hints = IndexMap::new(); + hints.insert( + "media".into(), + FieldHint { field_type: Some(FieldType::File), accept: Some("audio/*".into()), placeholder: None }, + ); + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + ..Default::default() + }; + let null = http_input_node(); + let nodes = vec![node("streamkit::http_input", Some(&null))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "field-hint-unknown-field"), + "Should not warn for default 'media' field: {warnings:?}" + ); + } + + #[test] + fn test_lint_field_hint_explicit_fields_array() { + let mut hints = IndexMap::new(); + hints.insert( + "prompt".into(), + FieldHint { field_type: Some(FieldType::Text), accept: None, placeholder: Some("Enter text".into()) }, + ); + let c = ClientSection { + input: Some(InputConfig { + input_type: InputType::FileUpload, + accept: Some("audio/*".into()), + asset_tags: None, + placeholder: None, + field_hints: Some(hints), + }), + output: Some(OutputConfig { output_type: OutputType::Audio }), + ..Default::default() + }; + let params = serde_json::json!({ + "fields": [ + { "name": "media" }, + { "name": "prompt" } + ] + }); + let nodes = vec![node("streamkit::http_input", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::OneShot, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "field-hint-unknown-field"), + "Should not warn when hint matches declared field: {warnings:?}" + ); + } + + // Rule 16 — publish-no-transport + #[test] + fn test_lint_publish_no_transport() { + let c = dynamic_client(); + let nodes: Vec> = vec![]; // no MoQ nodes + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!(warnings.iter().any(|w| w.rule == "publish-no-transport")); + } + + #[test] + fn test_lint_publish_with_peer_clean() { + let c = dynamic_client(); + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "input", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "publish-no-transport"), + "Should not warn when peer exists: {warnings:?}" + ); + } + + // Rule 17 — watch-no-transport + #[test] + fn test_lint_watch_no_transport() { + let c = ClientSection { + gateway_path: Some("/moq/test".into()), + watch: Some(WatchConfig { broadcast: "output".into(), audio: true, video: true }), + ..Default::default() + }; + let nodes: Vec> = vec![]; // no MoQ nodes + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!(warnings.iter().any(|w| w.rule == "watch-no-transport")); + } + + // Rule 18 — gateway-path-mismatch + #[test] + fn test_lint_gateway_path_mismatch() { + let c = ClientSection { + gateway_path: Some("/moq/wrong".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + ..Default::default() + }; + let params = serde_json::json!({ + "gateway_path": "/moq/correct", + "input_broadcast": "input" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "gateway-path-mismatch"), + "Should warn when gateway_path differs: {warnings:?}" + ); + } + + #[test] + fn test_lint_gateway_path_match_clean() { + let c = dynamic_client(); // gateway_path = /moq/test + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "input", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "gateway-path-mismatch"), + "Should not warn when gateway_path matches: {warnings:?}" + ); + } + + // Rule 19 — relay-url-mismatch + #[test] + fn test_lint_relay_url_mismatch() { + let c = ClientSection { + relay_url: Some("https://relay.example.com".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + ..Default::default() + }; + let params = serde_json::json!({ + "url": "https://other-relay.example.com", + "broadcast": "input" + }); + let nodes = vec![node("transport::moq::publisher", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "relay-url-mismatch"), + "Should warn when relay_url differs: {warnings:?}" + ); + } + + #[test] + fn test_lint_relay_url_match_clean() { + let c = ClientSection { + relay_url: Some("https://relay.example.com".into()), + publish: Some(PublishConfig { broadcast: "input".into(), audio: true, video: false }), + ..Default::default() + }; + let params = serde_json::json!({ + "url": "https://relay.example.com", + "broadcast": "input" + }); + let nodes = vec![node("transport::moq::subscriber", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "relay-url-mismatch"), + "Should not warn when relay_url matches: {warnings:?}" + ); + } + + // Rule 20 — broadcast-mismatch + #[test] + fn test_lint_broadcast_mismatch_publish() { + let c = ClientSection { + gateway_path: Some("/moq/test".into()), + publish: Some(PublishConfig { broadcast: "wrong_name".into(), audio: true, video: false }), + ..Default::default() + }; + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "camera", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "broadcast-mismatch"), + "Should warn when publish.broadcast doesn't match any node broadcast: {warnings:?}" + ); + } + + #[test] + fn test_lint_broadcast_mismatch_watch() { + let c = ClientSection { + gateway_path: Some("/moq/test".into()), + watch: Some(WatchConfig { broadcast: "wrong_name".into(), audio: true, video: true }), + ..Default::default() + }; + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "camera", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + warnings.iter().any(|w| w.rule == "broadcast-mismatch"), + "Should warn when watch.broadcast doesn't match any node broadcast: {warnings:?}" + ); + } + + #[test] + fn test_lint_broadcast_match_clean() { + let c = dynamic_client(); // publish=input, watch=output + let params = serde_json::json!({ + "gateway_path": "/moq/test", + "input_broadcast": "input", + "output_broadcast": "output" + }); + let nodes = vec![node("transport::moq::peer", Some(¶ms))]; + let warnings = lint_client_against_nodes(&c, EngineMode::Dynamic, &nodes); + assert!( + !warnings.iter().any(|w| w.rule == "broadcast-mismatch"), + "Should not warn when broadcast names match: {warnings:?}" + ); + } } diff --git a/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml b/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml index fe44c9f9..60c1860a 100644 --- a/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml +++ b/samples/loadtest/pipelines/oneshot_mixing_no_pacer.yml @@ -4,7 +4,6 @@ # Purpose: maximize throughput/CPU load by removing real-time pacer nodes. # This intentionally processes inputs as fast as possible. # -# skit:input_asset_tags=speech # name: "Loadtest: Audio Mixing (No Pacer)" diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts index fa6bc3df..b0df08ff 100644 --- a/ui/src/utils/clientSection.test.ts +++ b/ui/src/utils/clientSection.test.ts @@ -256,3 +256,216 @@ nodes: expect(parseClientFromYaml('{{invalid')).toBeNull(); }); }); + +// ----------------------------------------------------------------------- +// Monitor preview config derivation (Gap 4a) +// ----------------------------------------------------------------------- + +describe('deriveSettingsFromClient — monitor preview scenarios', () => { + it('derives watch-only settings for monitor preview (no publish)', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/transcoder', + publish: null, + watch: { broadcast: 'output', audio: true, video: false }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings.gatewayPath).toBe('/moq/transcoder'); + expect(settings.outputBroadcast).toBe('output'); + expect(settings.hasInputBroadcast).toBe(false); + expect(settings.outputsAudio).toBe(true); + expect(settings.outputsVideo).toBe(false); + }); + + it('derives audio+video output for compositor preview', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/compositor', + publish: { broadcast: 'camera', audio: true, video: true }, + watch: { broadcast: 'composited', audio: true, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings.outputsAudio).toBe(true); + expect(settings.outputsVideo).toBe(true); + expect(settings.outputBroadcast).toBe('composited'); + }); + + it('derives relay-url settings for external relay preview', () => { + const client: ClientSection = { + relay_url: 'https://relay.example.com', + gateway_path: null, + publish: null, + watch: { broadcast: 'preview', audio: false, video: true }, + input: null, + output: null, + }; + + const settings = deriveSettingsFromClient(client); + + expect(settings.relayUrl).toBe('https://relay.example.com'); + expect(settings.gatewayPath).toBeUndefined(); + expect(settings.outputsVideo).toBe(true); + expect(settings.outputsAudio).toBe(false); + }); +}); + +// ----------------------------------------------------------------------- +// ConvertView-driven pipeline classification (Gap 4b) +// ----------------------------------------------------------------------- + +describe('parseClientFromYaml — trigger vs none and output-type', () => { + it('parses trigger input type', () => { + const yaml = ` +client: + input: + type: trigger + output: + type: video +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + expect(client).not.toBeNull(); + expect(client?.input?.type).toBe('trigger'); + expect(client?.output?.type).toBe('video'); + }); + + it('parses none input type', () => { + const yaml = ` +client: + input: + type: none + output: + type: video +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + expect(client?.input?.type).toBe('none'); + }); + + it('identifies transcription output type for renderer selection', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: transcription +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('transcription'); + // ConvertView uses: isTranscriptionPipeline = client?.output?.type === 'transcription' + expect(client?.output?.type === 'transcription').toBe(true); + }); + + it('identifies audio output type for audio player renderer', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('audio'); + // ConvertView renders CustomAudioPlayer for audio output + expect(client?.output?.type === 'audio').toBe(true); + }); + + it('identifies video output type for video player renderer', () => { + const yaml = ` +client: + input: + type: none + output: + type: video +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('video'); + // ConvertView: isVideoPipeline = client?.output?.type === 'video' + const isNoInput = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + expect(isNoInput).toBe(true); + }); + + it('identifies json_stream output type for JsonStreamDisplay', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: json_stream +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + expect(client?.output?.type).toBe('json_stream'); + }); + + it('classifies text input as TTS pipeline', () => { + const yaml = ` +client: + input: + type: text + placeholder: "Enter text to convert to speech" + output: + type: audio +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + // ConvertView: isTTSPipeline = client?.input?.type === 'text' + expect(client?.input?.type).toBe('text'); + expect(client?.input?.type === 'text').toBe(true); + }); + + it('trigger and none both classify as no-input pipeline', () => { + for (const inputType of ['trigger', 'none']) { + const yaml = ` +client: + input: + type: ${inputType} + output: + type: audio +steps: + - kind: core::passthrough +`; + const client = parseClientFromYaml(yaml); + // ConvertView: isNoInputPipeline = client?.input?.type === 'none' || client?.input?.type === 'trigger' + const isNoInput = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + expect(isNoInput).toBe(true); + } + }); + + it('file_upload does NOT classify as no-input pipeline', () => { + const yaml = ` +client: + input: + type: file_upload + accept: "audio/*" + output: + type: audio +steps: + - kind: streamkit::http_input +`; + const client = parseClientFromYaml(yaml); + const isNoInput = client?.input?.type === 'none' || client?.input?.type === 'trigger'; + expect(isNoInput).toBe(false); + }); +}); From 5f1e25ce681545f8043404aa484dba52f7c4eacf Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:35:54 +0000 Subject: [PATCH 11/20] fix: narrow accept from audio/* to audio/ogg in all ogg-only pipelines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 10 oneshot pipelines that use ogg::demuxer → opus::decoder only accept Ogg/Opus input, not arbitrary audio formats. Change accept from the blanket 'audio/*' to the specific 'audio/ogg' MIME type so the browser file picker filters correctly. Co-Authored-By: Claudio Costa --- samples/pipelines/oneshot/double_volume.yml | 2 +- samples/pipelines/oneshot/dual_upload_mixing.yml | 2 +- samples/pipelines/oneshot/gain_filter_rust.yml | 2 +- samples/pipelines/oneshot/mixing.yml | 2 +- samples/pipelines/oneshot/sensevoice-stt.yml | 2 +- samples/pipelines/oneshot/speech_to_text.yml | 2 +- samples/pipelines/oneshot/speech_to_text_translate.yml | 2 +- samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml | 2 +- samples/pipelines/oneshot/vad-demo.yml | 2 +- samples/pipelines/oneshot/vad-filtered-stt.yml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/samples/pipelines/oneshot/double_volume.yml b/samples/pipelines/oneshot/double_volume.yml index b2e4b1b3..f919f158 100644 --- a/samples/pipelines/oneshot/double_volume.yml +++ b/samples/pipelines/oneshot/double_volume.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" output: type: audio steps: diff --git a/samples/pipelines/oneshot/dual_upload_mixing.yml b/samples/pipelines/oneshot/dual_upload_mixing.yml index 062ef25f..875a925f 100644 --- a/samples/pipelines/oneshot/dual_upload_mixing.yml +++ b/samples/pipelines/oneshot/dual_upload_mixing.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/gain_filter_rust.yml b/samples/pipelines/oneshot/gain_filter_rust.yml index b81121a0..dc975a63 100644 --- a/samples/pipelines/oneshot/gain_filter_rust.yml +++ b/samples/pipelines/oneshot/gain_filter_rust.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" output: type: audio steps: diff --git a/samples/pipelines/oneshot/mixing.yml b/samples/pipelines/oneshot/mixing.yml index 5b42e031..b111075f 100644 --- a/samples/pipelines/oneshot/mixing.yml +++ b/samples/pipelines/oneshot/mixing.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/sensevoice-stt.yml b/samples/pipelines/oneshot/sensevoice-stt.yml index 8f27d51b..4be20250 100644 --- a/samples/pipelines/oneshot/sensevoice-stt.yml +++ b/samples/pipelines/oneshot/sensevoice-stt.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/speech_to_text.yml b/samples/pipelines/oneshot/speech_to_text.yml index 6c2441a8..c318b0ac 100644 --- a/samples/pipelines/oneshot/speech_to_text.yml +++ b/samples/pipelines/oneshot/speech_to_text.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/speech_to_text_translate.yml b/samples/pipelines/oneshot/speech_to_text_translate.yml index c4a29be0..a8583d96 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate.yml @@ -8,7 +8,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml index 3eb28a78..11954c1f 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml @@ -18,7 +18,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/vad-demo.yml b/samples/pipelines/oneshot/vad-demo.yml index 0833504e..bfc2e144 100644 --- a/samples/pipelines/oneshot/vad-demo.yml +++ b/samples/pipelines/oneshot/vad-demo.yml @@ -8,7 +8,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/vad-filtered-stt.yml b/samples/pipelines/oneshot/vad-filtered-stt.yml index 6a8af170..5834b8bb 100644 --- a/samples/pipelines/oneshot/vad-filtered-stt.yml +++ b/samples/pipelines/oneshot/vad-filtered-stt.yml @@ -8,7 +8,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/*" + accept: "audio/ogg" asset_tags: - speech output: From c2efd26b626617cd6ae758a1fa61608b665d5fcb Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:39:06 +0000 Subject: [PATCH 12/20] fix: use audio/opus instead of audio/ogg for precise MIME type audio/opus is the correct MIME type for Opus-encoded audio, which is what all these pipelines actually decode via opus::decoder. Co-Authored-By: Claudio Costa --- samples/pipelines/oneshot/double_volume.yml | 2 +- samples/pipelines/oneshot/dual_upload_mixing.yml | 2 +- samples/pipelines/oneshot/gain_filter_rust.yml | 2 +- samples/pipelines/oneshot/mixing.yml | 2 +- samples/pipelines/oneshot/sensevoice-stt.yml | 2 +- samples/pipelines/oneshot/speech_to_text.yml | 2 +- samples/pipelines/oneshot/speech_to_text_translate.yml | 2 +- samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml | 2 +- samples/pipelines/oneshot/vad-demo.yml | 2 +- samples/pipelines/oneshot/vad-filtered-stt.yml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/samples/pipelines/oneshot/double_volume.yml b/samples/pipelines/oneshot/double_volume.yml index f919f158..ac7b48dc 100644 --- a/samples/pipelines/oneshot/double_volume.yml +++ b/samples/pipelines/oneshot/double_volume.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" output: type: audio steps: diff --git a/samples/pipelines/oneshot/dual_upload_mixing.yml b/samples/pipelines/oneshot/dual_upload_mixing.yml index 875a925f..d15c5dd6 100644 --- a/samples/pipelines/oneshot/dual_upload_mixing.yml +++ b/samples/pipelines/oneshot/dual_upload_mixing.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/gain_filter_rust.yml b/samples/pipelines/oneshot/gain_filter_rust.yml index dc975a63..44bd3f77 100644 --- a/samples/pipelines/oneshot/gain_filter_rust.yml +++ b/samples/pipelines/oneshot/gain_filter_rust.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" output: type: audio steps: diff --git a/samples/pipelines/oneshot/mixing.yml b/samples/pipelines/oneshot/mixing.yml index b111075f..069edcaf 100644 --- a/samples/pipelines/oneshot/mixing.yml +++ b/samples/pipelines/oneshot/mixing.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/sensevoice-stt.yml b/samples/pipelines/oneshot/sensevoice-stt.yml index 4be20250..57cd82a4 100644 --- a/samples/pipelines/oneshot/sensevoice-stt.yml +++ b/samples/pipelines/oneshot/sensevoice-stt.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/speech_to_text.yml b/samples/pipelines/oneshot/speech_to_text.yml index c318b0ac..8d797f5f 100644 --- a/samples/pipelines/oneshot/speech_to_text.yml +++ b/samples/pipelines/oneshot/speech_to_text.yml @@ -4,7 +4,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/speech_to_text_translate.yml b/samples/pipelines/oneshot/speech_to_text_translate.yml index a8583d96..efbe1697 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate.yml @@ -8,7 +8,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml index 11954c1f..5f489f65 100644 --- a/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml +++ b/samples/pipelines/oneshot/speech_to_text_translate_helsinki.yml @@ -18,7 +18,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/vad-demo.yml b/samples/pipelines/oneshot/vad-demo.yml index bfc2e144..dad8056b 100644 --- a/samples/pipelines/oneshot/vad-demo.yml +++ b/samples/pipelines/oneshot/vad-demo.yml @@ -8,7 +8,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: diff --git a/samples/pipelines/oneshot/vad-filtered-stt.yml b/samples/pipelines/oneshot/vad-filtered-stt.yml index 5834b8bb..e423285e 100644 --- a/samples/pipelines/oneshot/vad-filtered-stt.yml +++ b/samples/pipelines/oneshot/vad-filtered-stt.yml @@ -8,7 +8,7 @@ mode: oneshot client: input: type: file_upload - accept: "audio/ogg" + accept: "audio/opus" asset_tags: - speech output: From 5360c173e290970952382f70a58e78d89c4cd824 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:42:18 +0000 Subject: [PATCH 13/20] fix: consolidate double YAML parse in ConvertView Parse pipelineYaml once into a shared useMemo and derive both the client section (via extractClientFromParsed) and http_input fields (via deriveHttpInputFieldsFromParsed) from the same parsed object. Previously parseClientFromYaml and deriveHttpInputFields each independently called load(yaml), parsing the same string twice on every pipelineYaml change. Also adds extractClientFromParsed() to clientSection.ts with 4 unit tests. Co-Authored-By: Claudio Costa --- ui/src/utils/clientSection.test.ts | 28 ++++++++++++++++++++++++++++ ui/src/utils/clientSection.ts | 15 +++++++++++++-- ui/src/views/ConvertView.tsx | 29 +++++++++++++++++++++-------- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts index b0df08ff..929ddd96 100644 --- a/ui/src/utils/clientSection.test.ts +++ b/ui/src/utils/clientSection.test.ts @@ -8,11 +8,39 @@ import type { ClientSection } from '@/types/types'; import { deriveSettingsFromClient, + extractClientFromParsed, extractClientSection, parseAcceptToFormats, parseClientFromYaml, } from './clientSection'; +describe('extractClientFromParsed', () => { + it('returns null for null input', () => { + expect(extractClientFromParsed(null)).toBeNull(); + }); + + it('returns null for undefined input', () => { + expect(extractClientFromParsed(undefined)).toBeNull(); + }); + + it('returns null when parsed object has no client key', () => { + expect(extractClientFromParsed({ nodes: {}, steps: [] })).toBeNull(); + }); + + it('extracts client section from a pre-parsed object', () => { + const client: ClientSection = { + relay_url: null, + gateway_path: '/moq/test', + publish: null, + watch: null, + input: { type: 'file_upload', accept: 'audio/opus' }, + output: { type: 'audio' }, + }; + const parsed = { nodes: {}, client }; + expect(extractClientFromParsed(parsed)).toBe(client); + }); +}); + describe('extractClientSection', () => { it('returns null for null pipeline', () => { expect(extractClientSection(null)).toBeNull(); diff --git a/ui/src/utils/clientSection.ts b/ui/src/utils/clientSection.ts index f577aab3..56cfc4f7 100644 --- a/ui/src/utils/clientSection.ts +++ b/ui/src/utils/clientSection.ts @@ -14,6 +14,18 @@ export function extractClientSection(pipeline: Pipeline | null | undefined): Cli return pipeline?.client ?? null; } +/** + * Extracts the `client` section from an already-parsed YAML object. + * Use this when you have already called `load()` and want to avoid + * parsing the same YAML string again. + */ +export function extractClientFromParsed( + parsed: Record | null | undefined +): ClientSection | null { + if (!parsed || typeof parsed !== 'object') return null; + return (parsed.client as ClientSection) ?? null; +} + /** * Parses a raw pipeline YAML string and extracts the `client` section. * Returns `null` if the YAML is invalid or has no client section. @@ -21,8 +33,7 @@ export function extractClientSection(pipeline: Pipeline | null | undefined): Cli export function parseClientFromYaml(yamlContent: string): ClientSection | null { try { const parsed = load(yamlContent) as Record | null; - if (!parsed || typeof parsed !== 'object') return null; - return (parsed.client as ClientSection) ?? null; + return extractClientFromParsed(parsed); } catch { return null; } diff --git a/ui/src/views/ConvertView.tsx b/ui/src/views/ConvertView.tsx index bed95356..547be603 100644 --- a/ui/src/views/ConvertView.tsx +++ b/ui/src/views/ConvertView.tsx @@ -39,7 +39,11 @@ import { } from '@/services/converter'; import { listSamples } from '@/services/samples'; import { ensureSchemasLoaded, useSchemaStore } from '@/stores/schemaStore'; -import { parseAcceptToFormats, parseClientFromYaml } from '@/utils/clientSection'; +import { + extractClientFromParsed, + parseAcceptToFormats, + parseClientFromYaml, +} from '@/utils/clientSection'; import { viewsLogger } from '@/utils/logger'; import { orderSamplePipelinesSystemFirst } from '@/utils/samplePipelineOrdering'; import { injectFileReadNode } from '@/utils/yamlPipeline'; @@ -89,11 +93,10 @@ const extractFieldsFromNode = ( return fallback ? [{ name: fallback, required: defaultField ? false : true }] : []; }; -const deriveHttpInputFields = ( - yaml: string +const deriveHttpInputFieldsFromParsed = ( + parsed: Record | null | undefined ): { fields: HttpInputField[]; hasHttpInput: boolean } => { try { - const parsed = loadYaml(yaml) as { nodes?: unknown; steps?: unknown } | null; if (!parsed || typeof parsed !== 'object') return { fields: [], hasHttpInput: false }; if (isRecord(parsed.nodes)) { @@ -582,8 +585,18 @@ const ConvertView: React.FC = () => { const [msePlaybackError, setMsePlaybackError] = useState(null); const [mseFallbackLoading, setMseFallbackLoading] = useState(false); - // Derive pipeline characteristics from declarative client section - const client = useMemo(() => parseClientFromYaml(pipelineYaml), [pipelineYaml]); + // Parse pipeline YAML once — derive both client section and http_input fields + // from the same parsed object to avoid double-parsing. + const parsedPipelineYaml = useMemo(() => { + try { + const parsed = loadYaml(pipelineYaml) as Record | null; + return parsed && typeof parsed === 'object' ? parsed : null; + } catch { + return null; + } + }, [pipelineYaml]); + + const client = useMemo(() => extractClientFromParsed(parsedPipelineYaml), [parsedPipelineYaml]); const isTranscriptionPipeline = client?.output?.type === 'transcription'; const isTTSPipeline = client?.input?.type === 'text'; const isNoInputPipeline = client?.input?.type === 'none' || client?.input?.type === 'trigger'; @@ -713,7 +726,7 @@ const ConvertView: React.FC = () => { // Track http_input fields for multi-upload pipelines useEffect(() => { - const { fields, hasHttpInput: hasHttp } = deriveHttpInputFields(pipelineYaml); + const { fields, hasHttpInput: hasHttp } = deriveHttpInputFieldsFromParsed(parsedPipelineYaml); setHasHttpInput(hasHttp); setHttpInputFields(fields); setFieldUploads((prev) => { @@ -723,7 +736,7 @@ const ConvertView: React.FC = () => { }); return next; }); - }, [pipelineYaml]); + }, [parsedPipelineYaml]); // Force playback mode for transcription/TTS pipelines useEffect(() => { From 79e4beae569b8cbca766e3d378cdfd3b349d41de Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:44:08 +0000 Subject: [PATCH 14/20] fix: add missing InputConfig fields in extractClientFromParsed test Co-Authored-By: Claudio Costa --- ui/src/utils/clientSection.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts index 929ddd96..96079e73 100644 --- a/ui/src/utils/clientSection.test.ts +++ b/ui/src/utils/clientSection.test.ts @@ -33,7 +33,13 @@ describe('extractClientFromParsed', () => { gateway_path: '/moq/test', publish: null, watch: null, - input: { type: 'file_upload', accept: 'audio/opus' }, + input: { + type: 'file_upload', + accept: 'audio/opus', + asset_tags: null, + placeholder: null, + field_hints: null, + }, output: { type: 'audio' }, }; const parsed = { nodes: {}, client }; From 55da7c90395341296c94d841164e01f46e6d65d9 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:46:21 +0000 Subject: [PATCH 15/20] style: apply rustfmt to yaml.rs Co-Authored-By: Claudio Costa --- crates/api/src/yaml.rs | 56 ++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/crates/api/src/yaml.rs b/crates/api/src/yaml.rs index 912398f6..e9b02ff9 100644 --- a/crates/api/src/yaml.rs +++ b/crates/api/src/yaml.rs @@ -857,7 +857,9 @@ pub fn lint_client_against_nodes( for node in nodes.iter().filter(|n| n.kind == "streamkit::http_input") { if let Some(params) = node.params { // Single field: { field: { name: "foo" } } - if let Some(name) = params.get("field").and_then(|f| f.get("name")).and_then(|n| n.as_str()) { + if let Some(name) = + params.get("field").and_then(|f| f.get("name")).and_then(|n| n.as_str()) + { declared_fields.push(name.to_string()); } // Multi field: { fields: [{ name: "foo" }, { name: "bar" }] } @@ -871,9 +873,9 @@ pub fn lint_client_against_nodes( } // http_input with no params has a single default field named "media" if (node.params.is_none() - || node.params.is_none_or(|p| { - p.get("field").is_none() && p.get("fields").is_none() - })) + || node + .params + .is_none_or(|p| p.get("field").is_none() && p.get("fields").is_none())) && !declared_fields.contains(&"media".to_string()) { declared_fields.push("media".to_string()); @@ -924,16 +926,10 @@ pub fn lint_client_against_nodes( let peer_gateway_paths: Vec<&str> = nodes .iter() .filter(|n| n.kind == "transport::moq::peer") - .filter_map(|n| { - n.params - .and_then(|p| p.get("gateway_path")) - .and_then(|v| v.as_str()) - }) + .filter_map(|n| n.params.and_then(|p| p.get("gateway_path")).and_then(|v| v.as_str())) .collect(); - if !peer_gateway_paths.is_empty() - && !peer_gateway_paths.iter().any(|gw| gw == client_gw) - { + if !peer_gateway_paths.is_empty() && !peer_gateway_paths.iter().any(|gw| gw == client_gw) { warnings.push(ClientLintWarning { rule: "gateway-path-mismatch", message: format!( @@ -949,14 +945,9 @@ pub fn lint_client_against_nodes( let node_urls: Vec<&str> = nodes .iter() .filter(|n| { - n.kind == "transport::moq::publisher" - || n.kind == "transport::moq::subscriber" - }) - .filter_map(|n| { - n.params - .and_then(|p| p.get("url")) - .and_then(|v| v.as_str()) + n.kind == "transport::moq::publisher" || n.kind == "transport::moq::subscriber" }) + .filter_map(|n| n.params.and_then(|p| p.get("url")).and_then(|v| v.as_str())) .collect(); if !node_urls.is_empty() && !node_urls.iter().any(|u| u == client_url) { @@ -983,13 +974,13 @@ pub fn lint_client_against_nodes( if let Some(b) = params.get("output_broadcast").and_then(|v| v.as_str()) { node_broadcasts.push(b); } - } + }, "transport::moq::publisher" | "transport::moq::subscriber" => { if let Some(b) = params.get("broadcast").and_then(|v| v.as_str()) { node_broadcasts.push(b); } - } - _ => {} + }, + _ => {}, } } } @@ -1011,8 +1002,7 @@ pub fn lint_client_against_nodes( } } if let Some(ref watch) = client.watch { - if !watch.broadcast.is_empty() - && !node_broadcasts.iter().any(|b| *b == watch.broadcast) + if !watch.broadcast.is_empty() && !node_broadcasts.iter().any(|b| *b == watch.broadcast) { warnings.push(ClientLintWarning { rule: "broadcast-mismatch", @@ -2065,7 +2055,11 @@ client: let mut hints = IndexMap::new(); hints.insert( "media".into(), - FieldHint { field_type: Some(FieldType::File), accept: Some("audio/*".into()), placeholder: None }, + FieldHint { + field_type: Some(FieldType::File), + accept: Some("audio/*".into()), + placeholder: None, + }, ); let c = ClientSection { input: Some(InputConfig { @@ -2092,7 +2086,11 @@ client: let mut hints = IndexMap::new(); hints.insert( "prompt".into(), - FieldHint { field_type: Some(FieldType::Text), accept: None, placeholder: Some("Enter text".into()) }, + FieldHint { + field_type: Some(FieldType::Text), + accept: None, + placeholder: Some("Enter text".into()), + }, ); let c = ClientSection { input: Some(InputConfig { @@ -2237,7 +2235,11 @@ client: fn test_lint_broadcast_mismatch_publish() { let c = ClientSection { gateway_path: Some("/moq/test".into()), - publish: Some(PublishConfig { broadcast: "wrong_name".into(), audio: true, video: false }), + publish: Some(PublishConfig { + broadcast: "wrong_name".into(), + audio: true, + video: false, + }), ..Default::default() }; let params = serde_json::json!({ From 5d900730d13542a3d9a69d4ee0560beea444ce3c Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:48:57 +0000 Subject: [PATCH 16/20] refactor: move deriveSettingsFromClient to moqPeerSettings.ts Eliminates the duplicated inline return type by co-locating the function with the MoqPeerSettings interface it returns. No more circular import workaround needed. Co-Authored-By: Claudio Costa --- ui/src/utils/clientSection.test.ts | 2 +- ui/src/utils/clientSection.ts | 31 ------------------------------ ui/src/utils/moqPeerSettings.ts | 21 +++++++++++++++++++- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts index 96079e73..4df6469d 100644 --- a/ui/src/utils/clientSection.test.ts +++ b/ui/src/utils/clientSection.test.ts @@ -7,12 +7,12 @@ import { describe, expect, it } from 'vitest'; import type { ClientSection } from '@/types/types'; import { - deriveSettingsFromClient, extractClientFromParsed, extractClientSection, parseAcceptToFormats, parseClientFromYaml, } from './clientSection'; +import { deriveSettingsFromClient } from './moqPeerSettings'; describe('extractClientFromParsed', () => { it('returns null for null input', () => { diff --git a/ui/src/utils/clientSection.ts b/ui/src/utils/clientSection.ts index 56cfc4f7..6ff2273e 100644 --- a/ui/src/utils/clientSection.ts +++ b/ui/src/utils/clientSection.ts @@ -63,34 +63,3 @@ export function parseAcceptToFormats(accept: string | null | undefined): string[ } return formats.length > 0 ? formats : null; } - -/** - * Derives `MoqPeerSettings`-shaped data from a declarative `ClientSection`. - * - * The return type mirrors the `MoqPeerSettings` interface defined in - * `moqPeerSettings.ts`. We avoid importing that interface here to prevent - * a circular module dependency. - */ -export function deriveSettingsFromClient(client: ClientSection): { - gatewayPath?: string; - relayUrl?: string; - inputBroadcast?: string; - outputBroadcast?: string; - hasInputBroadcast: boolean; - needsAudioInput: boolean; - needsVideoInput: boolean; - outputsAudio: boolean; - outputsVideo: boolean; -} { - return { - gatewayPath: client.gateway_path ?? undefined, - relayUrl: client.relay_url ?? undefined, - inputBroadcast: client.publish?.broadcast, - outputBroadcast: client.watch?.broadcast, - hasInputBroadcast: Boolean(client.publish), - needsAudioInput: client.publish?.audio ?? false, - needsVideoInput: client.publish?.video ?? false, - outputsAudio: client.watch?.audio ?? false, - outputsVideo: client.watch?.video ?? false, - }; -} diff --git a/ui/src/utils/moqPeerSettings.ts b/ui/src/utils/moqPeerSettings.ts index a60ce1a1..be062c61 100644 --- a/ui/src/utils/moqPeerSettings.ts +++ b/ui/src/utils/moqPeerSettings.ts @@ -2,7 +2,9 @@ // // SPDX-License-Identifier: MPL-2.0 -import { deriveSettingsFromClient, parseClientFromYaml } from './clientSection'; +import type { ClientSection } from '@/types/types'; + +import { parseClientFromYaml } from './clientSection'; export interface MoqPeerSettings { gatewayPath?: string; @@ -22,6 +24,23 @@ export interface MoqPeerSettings { outputsVideo: boolean; } +/** + * Derives `MoqPeerSettings` from a declarative `ClientSection`. + */ +export function deriveSettingsFromClient(client: ClientSection): MoqPeerSettings { + return { + gatewayPath: client.gateway_path ?? undefined, + relayUrl: client.relay_url ?? undefined, + inputBroadcast: client.publish?.broadcast, + outputBroadcast: client.watch?.broadcast, + hasInputBroadcast: Boolean(client.publish), + needsAudioInput: client.publish?.audio ?? false, + needsVideoInput: client.publish?.video ?? false, + outputsAudio: client.watch?.audio ?? false, + outputsVideo: client.watch?.video ?? false, + }; +} + /** * Extracts MoQ peer settings from a pipeline YAML string by reading the * declarative `client` section. From 81d615c5991c13762b0932c26acbbaae013727c7 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Sun, 22 Mar 2026 19:55:59 +0000 Subject: [PATCH 17/20] fix: add node-graph fallback in useMonitorPreview and fix json test type - useMonitorPreview now falls back to scanning the pipeline's node graph (transport::moq::peer params + connection graph) when pipeline.client is null. This preserves monitor preview for interactively-created sessions that don't have a client section. - Extracted deriveMoqConfigFromNodes() helper to keep callback complexity within lint thresholds. - Fixed test that used non-existent 'json_stream' output type; the correct Rust enum value is 'json'. Co-Authored-By: Claudio Costa --- ui/src/hooks/useMonitorPreview.ts | 78 +++++++++++++++++++++++++----- ui/src/utils/clientSection.test.ts | 6 +-- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/ui/src/hooks/useMonitorPreview.ts b/ui/src/hooks/useMonitorPreview.ts index d57f5283..29f49765 100644 --- a/ui/src/hooks/useMonitorPreview.ts +++ b/ui/src/hooks/useMonitorPreview.ts @@ -18,6 +18,45 @@ import { useStreamStore } from '@/stores/streamStore'; import type { Pipeline } from '@/types/types'; import { updateUrlPath } from '@/utils/moqPeerSettings'; +interface PreviewMoqConfig { + gatewayPath?: string; + outputBroadcast?: string; + outputsAudio: boolean; + outputsVideo: boolean; +} + +/** + * Derives MoQ preview configuration from the pipeline's node graph. + * Used as a fallback for interactively-created sessions that don't have + * a `client` section. + */ +function deriveMoqConfigFromNodes(pipeline: Pipeline): PreviewMoqConfig { + const config: PreviewMoqConfig = { outputsAudio: true, outputsVideo: true }; + + const moqEntry = Object.entries(pipeline.nodes).find( + ([, n]) => n.kind === 'transport::moq::peer' && n.params + ); + if (!moqEntry) return config; + + const [moqNodeName, moqNode] = moqEntry; + const params = moqNode.params as Record; + config.gatewayPath = params.gateway_path as string | undefined; + config.outputBroadcast = params.output_broadcast as string | undefined; + + // Detect media types from connection graph + config.outputsAudio = false; + config.outputsVideo = false; + for (const conn of pipeline.connections) { + if (conn.to_node !== moqNodeName) continue; + const sourceNode = pipeline.nodes[conn.from_node]; + if (!sourceNode?.kind) continue; + if (sourceNode.kind.startsWith('audio::')) config.outputsAudio = true; + else if (sourceNode.kind.startsWith('video::')) config.outputsVideo = true; + } + + return config; +} + export interface UseMonitorPreviewReturn { isPreviewConnected: boolean; handleStartPreview: () => Promise; @@ -80,23 +119,36 @@ export function useMonitorPreview( } // Read gateway_path and output_broadcast from the pipeline's client section. + // Fall back to scanning the node graph for interactively-created sessions + // that don't have a client section. const client = pipeline?.client ?? null; + let gatewayPath: string | undefined; + let outputBroadcast: string | undefined; + let outputsAudio = true; + let outputsVideo = true; + if (client) { - if (client.gateway_path) { - const currentUrl = useStreamStore.getState().serverUrl; - if (currentUrl) { - previewSetServerUrl(updateUrlPath(currentUrl, client.gateway_path)); - } - } - if (client.watch?.broadcast) { - previewSetOutputBroadcast(client.watch.broadcast); - } + gatewayPath = client.gateway_path ?? undefined; + outputBroadcast = client.watch?.broadcast; + outputsAudio = client.watch?.audio ?? true; + outputsVideo = client.watch?.video ?? true; + } else if (pipeline) { + const fallback = deriveMoqConfigFromNodes(pipeline); + gatewayPath = fallback.gatewayPath; + outputBroadcast = fallback.outputBroadcast; + outputsAudio = fallback.outputsAudio; + outputsVideo = fallback.outputsVideo; } - // Media types default to both enabled unless the client section - // explicitly declares which types the pipeline outputs. - const outputsAudio = client?.watch?.audio ?? true; - const outputsVideo = client?.watch?.video ?? true; + if (gatewayPath) { + const currentUrl = useStreamStore.getState().serverUrl; + if (currentUrl) { + previewSetServerUrl(updateUrlPath(currentUrl, gatewayPath)); + } + } + if (outputBroadcast) { + previewSetOutputBroadcast(outputBroadcast); + } previewSetPipelineOutputTypes(outputsAudio, outputsVideo); await previewConnect(); diff --git a/ui/src/utils/clientSection.test.ts b/ui/src/utils/clientSection.test.ts index 4df6469d..d182e9c3 100644 --- a/ui/src/utils/clientSection.test.ts +++ b/ui/src/utils/clientSection.test.ts @@ -437,19 +437,19 @@ steps: expect(isNoInput).toBe(true); }); - it('identifies json_stream output type for JsonStreamDisplay', () => { + it('identifies json output type for JsonStreamDisplay', () => { const yaml = ` client: input: type: file_upload accept: "audio/*" output: - type: json_stream + type: json steps: - kind: streamkit::http_input `; const client = parseClientFromYaml(yaml); - expect(client?.output?.type).toBe('json_stream'); + expect(client?.output?.type).toBe('json'); }); it('classifies text input as TTS pipeline', () => { From ee2b44d7bee6ce764f89c75a1f0f73ee87ad469d Mon Sep 17 00:00:00 2001 From: "staging-devin-ai-integration[bot]" <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 15:36:07 +0100 Subject: [PATCH 18/20] fix: cancel in-flight connect on disconnect to prevent second-attempt bug (#182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: cancel in-flight connect on disconnect to prevent second-attempt bug The core issue: disconnect() during an in-flight performConnect() could not cancel it. The local 'attempt' objects (Camera, Connection, etc.) lived in performConnect's stack frame and were never cleaned up by disconnect()'s cleanupConnectAttempt() call (which only saw the store's null refs). When the user started a new pipeline, performConnect₁ would eventually complete and overwrite the store — clobbering performConnect₂'s state, leaking resources, or flipping the UI back to 'disconnected'. Changes: - Add AbortController to stream store; abort on disconnect() and connect() - Thread AbortSignal through waitForSignalValue, waitForBroadcastAnnouncement, setupMediaSources, setupPublishPath, and performConnect - Aborted performConnect silently discards its attempt (no store writes) - Add connectingStep state for granular UX feedback during connect phases (devices → relay → pipeline) - Surface auto-connect errors in StreamView instead of silently logging - Extract createConnectionAndHealth() and connectWatchPath() helpers to keep performConnect under the max-statements lint threshold - Add tests for abort signal cancellation in waitForSignalValue and disconnect-during-connect in streamStore Co-Authored-By: Claudio Costa * style: format StreamView.tsx with prettier Co-Authored-By: Claudio Costa * fix: address review — reset new fields in beforeEach, event-driven abort in waitForBroadcastAnnouncement - Add connectAbort and connectingStep to the beforeEach setState reset to prevent test pollution across runs. - Replace polling abort check in waitForBroadcastAnnouncement with an event-driven abortPromise in Promise.race so abort fires immediately even when announcements.next() is blocked. Co-Authored-By: Claudio Costa --------- Co-authored-by: StreamKit Devin Co-authored-by: Claudio Costa --- ui/src/stores/streamStore.test.ts | 36 ++++ ui/src/stores/streamStore.ts | 23 ++- ui/src/stores/streamStoreHelpers.test.ts | 47 +++++ ui/src/stores/streamStoreHelpers.ts | 214 +++++++++++++++++------ ui/src/views/StreamView.tsx | 24 ++- 5 files changed, 288 insertions(+), 56 deletions(-) diff --git a/ui/src/stores/streamStore.test.ts b/ui/src/stores/streamStore.test.ts index 4a40674c..3ea26272 100644 --- a/ui/src/stores/streamStore.test.ts +++ b/ui/src/stores/streamStore.test.ts @@ -98,6 +98,8 @@ describe('streamStore', () => { pipelineOutputsVideo: true, errorMessage: '', configLoaded: false, + connectAbort: null, + connectingStep: '', activeSessionId: null, activeSessionName: null, activePipelineName: null, @@ -149,6 +151,12 @@ describe('streamStore', () => { expect(state.activePipelineName).toBeNull(); }); + it('should have null connectAbort and empty connectingStep initially', () => { + const state = useStreamStore.getState(); + expect(state.connectAbort).toBeNull(); + expect(state.connectingStep).toBe(''); + }); + it('should have null MoQ references initially', () => { const state = useStreamStore.getState(); expect(state.publish).toBeNull(); @@ -497,6 +505,34 @@ describe('streamStore', () => { expect(useStreamStore.getState().status).toBe('disconnected'); }); + + it('should abort in-flight connect attempt on disconnect', () => { + const abort = new AbortController(); + const abortSpy = vi.spyOn(abort, 'abort'); + useStreamStore.setState({ + status: 'connecting', + connectAbort: abort, + }); + + useStreamStore.getState().disconnect(); + + expect(abortSpy).toHaveBeenCalled(); + const state = useStreamStore.getState(); + expect(state.status).toBe('disconnected'); + expect(state.connectAbort).toBeNull(); + expect(state.connectingStep).toBe(''); + }); + + it('should clear connectingStep on disconnect', () => { + useStreamStore.setState({ + status: 'connecting', + connectingStep: 'devices', + }); + + useStreamStore.getState().disconnect(); + + expect(useStreamStore.getState().connectingStep).toBe(''); + }); }); describe('connection state machine', () => { diff --git a/ui/src/stores/streamStore.ts b/ui/src/stores/streamStore.ts index 082268d7..c9cbc011 100644 --- a/ui/src/stores/streamStore.ts +++ b/ui/src/stores/streamStore.ts @@ -69,6 +69,11 @@ interface StreamState { activeSessionName: string | null; activePipelineName: string | null; + // Connect lifecycle — abort controller for the in-flight performConnect call + connectAbort: AbortController | null; + /** Human-readable label for the current phase of a connect attempt. */ + connectingStep: string; + // MoQ references (stored but not serialized) publish: Publish.Broadcast | null; watch: Watch.Broadcast | null; @@ -148,6 +153,8 @@ export const useStreamStore = create((set, get) => ({ isExternalRelay: false, errorMessage: '', configLoaded: false, + connectAbort: null, + connectingStep: '', // Active session state activeSessionId: null, @@ -235,7 +242,15 @@ export const useStreamStore = create((set, get) => ({ return false; } + // Abort any in-flight connect attempt so it doesn't later overwrite + // our state. This is defensive — normally disconnect() already aborted, + // but a rapid connect→connect without disconnect can still race. + state.connectAbort?.abort(); + + const abort = new AbortController(); set({ + connectAbort: abort, + connectingStep: '', status: 'connecting', errorMessage: '', watchStatus: decision.shouldWatch ? 'loading' : 'disabled', @@ -243,17 +258,23 @@ export const useStreamStore = create((set, get) => ({ cameraStatus: decision.shouldPublish && state.pipelineNeedsVideo ? 'requesting' : 'disabled', }); - return performConnect(state, decision, get, set); + return performConnect(state, decision, get, set, abort.signal); }, disconnect: () => { const state = get(); + // Cancel any in-flight connect attempt so its catch/success path + // doesn't later overwrite the freshly-reset store. + state.connectAbort?.abort(); + // Reuse the same teardown logic used when a connect attempt fails. cleanupConnectAttempt(state); set({ status: 'disconnected', + connectAbort: null, + connectingStep: '', isMicEnabled: false, micStatus: 'disabled', isCameraEnabled: false, diff --git a/ui/src/stores/streamStoreHelpers.test.ts b/ui/src/stores/streamStoreHelpers.test.ts index d3ca93a2..45f0e73b 100644 --- a/ui/src/stores/streamStoreHelpers.test.ts +++ b/ui/src/stores/streamStoreHelpers.test.ts @@ -299,6 +299,53 @@ describe('waitForSignalValue', () => { vi.useRealTimers(); }); + + it('should reject immediately when abortSignal is already aborted', async () => { + const signal = createMockSignal(0); + const abort = new AbortController(); + abort.abort(); + + await expect( + waitForSignalValue(signal, (v) => v > 0, 5_000, 'timeout', abort.signal) + ).rejects.toThrow('Aborted'); + }); + + it('should reject with AbortError when abortSignal fires during wait', async () => { + const signal = createMockSignal(0); + const abort = new AbortController(); + + const promise = waitForSignalValue(signal, (v) => v > 0, 5_000, 'timeout', abort.signal); + + // Abort before the signal value changes + abort.abort(); + + await expect(promise).rejects.toThrow('Aborted'); + }); + + it('should not reject on abort if predicate already matched', async () => { + const signal = createMockSignal(42); + const abort = new AbortController(); + + // Predicate matches initial value — resolves synchronously before abort + const value = await waitForSignalValue(signal, (v) => v === 42, 5_000, 'timeout', abort.signal); + expect(value).toBe(42); + + // Aborting after resolution should be harmless + abort.abort(); + }); + + it('should clean up subscription when abortSignal fires', async () => { + const signal = createMockSignal(0); + const abort = new AbortController(); + + const promise = waitForSignalValue(signal, (v) => v > 100, 5_000, 'timeout', abort.signal); + abort.abort(); + + await expect(promise).rejects.toThrow('Aborted'); + + // After abort, emitting new values should be harmless (no dangling listeners) + expect(() => signal.set(200)).not.toThrow(); + }); }); // --------------------------------------------------------------------------- diff --git a/ui/src/stores/streamStoreHelpers.ts b/ui/src/stores/streamStoreHelpers.ts index b80e7e6f..97244030 100644 --- a/ui/src/stores/streamStoreHelpers.ts +++ b/ui/src/stores/streamStoreHelpers.ts @@ -64,6 +64,9 @@ export interface ConnectableState { micStatus: MicStatus; cameraStatus: CameraStatus; watchStatus: WatchStatus; + /** Human-readable label for the current phase of a connect attempt + * (e.g. 'devices', 'relay', 'pipeline'). Empty when idle. */ + connectingStep: string; } type StateSetter = (partial: Partial) => void; @@ -89,8 +92,13 @@ export function waitForSignalValue( signal: Getter, predicate: (value: T) => boolean, timeoutMs: number, - timeoutMessage: string + timeoutMessage: string, + abortSignal?: AbortSignal ): Promise { + if (abortSignal?.aborted) { + return Promise.reject(new DOMException('Aborted', 'AbortError')); + } + const initial = signal.peek(); if (predicate(initial)) { return Promise.resolve(initial); @@ -98,15 +106,31 @@ export function waitForSignalValue( return new Promise((resolve, reject) => { let dispose: () => void = () => {}; - const timeoutId = setTimeout(() => { + + const cleanup = () => { + clearTimeout(timeoutId); dispose(); + }; + + const timeoutId = setTimeout(() => { + cleanup(); reject(new Error(timeoutMessage)); }, timeoutMs); + if (abortSignal) { + abortSignal.addEventListener( + 'abort', + () => { + cleanup(); + reject(new DOMException('Aborted', 'AbortError')); + }, + { once: true } + ); + } + dispose = signal.subscribe((value) => { if (predicate(value)) { - clearTimeout(timeoutId); - dispose(); + cleanup(); resolve(value); } }); @@ -122,20 +146,39 @@ export function waitForSignalValue( async function waitForBroadcastAnnouncement( connection: Hang.Moq.Connection.Reload, broadcastName: string, - timeoutMs = 15_000 + timeoutMs = 15_000, + abortSignal?: AbortSignal ): Promise { + if (abortSignal?.aborted) throw new DOMException('Aborted', 'AbortError'); const conn = connection.established.peek(); if (!conn) return; logger.info(`Waiting for broadcast '${broadcastName}' announcement...`); const announcements = conn.announced(); const deadline = Date.now() + timeoutMs; try { + // Build a promise that rejects when the abort signal fires so the + // Promise.race below reacts immediately instead of polling. + const abortPromise = abortSignal + ? new Promise((_, reject) => { + abortSignal.addEventListener( + 'abort', + () => reject(new DOMException('Aborted', 'AbortError')), + { once: true } + ); + }) + : null; + while (Date.now() < deadline) { const remaining = deadline - Date.now(); - const entry = await Promise.race([ + const racers: Promise[] = [ announcements.next(), new Promise((r) => setTimeout(() => r(null), remaining)), - ]); + ]; + if (abortPromise) racers.push(abortPromise); + + const entry = (await Promise.race(racers)) as Awaited< + ReturnType + > | null; if (!entry) break; if (entry.active && entry.path.toString() === broadcastName) { logger.info(`Broadcast '${broadcastName}' announced`); @@ -320,7 +363,8 @@ async function setupMediaSources( healthEffect: Effect, needsAudio: boolean, needsVideo: boolean, - set: StateSetter + set: StateSetter, + abortSignal?: AbortSignal ): Promise<{ microphone: Publish.Source.Microphone | null; camera: Publish.Source.Camera | null; @@ -346,7 +390,8 @@ async function setupMediaSources( camera.source, (v) => v !== undefined, 15_000, - 'Camera not available' + 'Camera not available', + abortSignal ); } catch (e) { shutdownMediaSource(camera); @@ -364,13 +409,20 @@ async function setupPublishPath( inputBroadcast: string, needsAudio: boolean, needsVideo: boolean, - set: StateSetter + set: StateSetter, + abortSignal?: AbortSignal ): Promise<{ microphone: Publish.Source.Microphone | null; camera: Publish.Source.Camera | null; publish: Publish.Broadcast; }> { - const { microphone, camera } = await setupMediaSources(healthEffect, needsAudio, needsVideo, set); + const { microphone, camera } = await setupMediaSources( + healthEffect, + needsAudio, + needsVideo, + set, + abortSignal + ); logger.info('Step 5: Creating publish broadcast'); const broadcastConfig: ConstructorParameters[0] = { @@ -398,7 +450,8 @@ async function setupPublishPath( publish.video.catalog, (v) => v !== undefined, 10_000, - 'Video encoder failed to initialize' + 'Video encoder failed to initialize', + abortSignal ); } catch (e) { publish.close(); @@ -499,32 +552,102 @@ function applyPublishResult( attempt.publish = result.publish; } +/** Create the MoQ connection and wire up the health-status sync effect. */ +function createConnectionAndHealth( + serverUrl: string, + moqToken: string, + get: () => ConnectableState, + set: StateSetter +): { connection: Hang.Moq.Connection.Reload; healthEffect: Effect } { + logger.info('Step 1: Creating connection to relay server'); + const url = new URL(serverUrl); + const jwt = moqToken.trim(); + if (jwt) { + url.searchParams.set('jwt', jwt); + } + + const connection = new Hang.Moq.Connection.Reload({ url, enabled: true }); + const healthEffect = new Effect(); + setupConnectionStatusSync(healthEffect, connection, get, set); + return { connection, healthEffect }; +} + +/** Wait for relay connection, optionally wait for broadcast announcement, then set up watch. */ +async function connectWatchPath( + attempt: ConnectAttempt, + state: ConnectableState, + decision: Extract, + set: StateSetter, + abortSignal: AbortSignal +): Promise { + set({ connectingStep: 'relay' }); + await waitForSignalValue( + attempt.connection!.established, + (value) => value !== undefined, + 12_000, + 'Timed out connecting to MoQ gateway.', + abortSignal + ); + + if (decision.shouldWatch) { + // When publishing to an external relay, the skit pipeline needs time to + // discover input tracks, build the graph, and start publishing output. + // Wait for the output broadcast to be announced on the relay before + // subscribing, otherwise the catalog subscribe gets RESET_STREAM. + // In gateway mode the skit server manages the peer connection directly, + // so no announcement polling is needed. + if (decision.shouldPublish && state.isExternalRelay) { + set({ connectingStep: 'pipeline' }); + await waitForBroadcastAnnouncement( + attempt.connection!, + state.outputBroadcast, + 15_000, + abortSignal + ); + } + + applyWatchResult( + attempt, + setupWatchPath( + attempt.healthEffect!, + attempt.connection!, + state.outputBroadcast, + state.pipelineOutputsAudio, + state.pipelineOutputsVideo, + set + ) + ); + } +} + /** Core connection logic extracted from the store for reduced complexity. */ export async function performConnect( state: ConnectableState, decision: Extract, get: () => ConnectableState & { outputBroadcast: string }, - set: StateSetter + set: StateSetter, + abortSignal: AbortSignal ): Promise { const attempt: ConnectAttempt = { ...NULL_MOQ_REFS }; try { - logger.info('Step 1: Creating connection to relay server'); - const url = new URL(decision.trimmedServerUrl); - const jwt = state.moqToken.trim(); - if (jwt) { - url.searchParams.set('jwt', jwt); - } + if (abortSignal.aborted) throw new DOMException('Aborted', 'AbortError'); - attempt.connection = new Hang.Moq.Connection.Reload({ url, enabled: true }); - attempt.healthEffect = new Effect(); - setupConnectionStatusSync(attempt.healthEffect, attempt.connection, get, set); + const { connection, healthEffect } = createConnectionAndHealth( + decision.trimmedServerUrl, + state.moqToken, + get, + set + ); + attempt.connection = connection; + attempt.healthEffect = healthEffect; // Set up publish BEFORE watch. For external relay pipelines (pub/sub), // the skit pipeline needs input data before it can publish output. // If we watch first, the subscribe to output/catalog.json fails with // RESET_STREAM because skit hasn't started publishing yet. if (decision.shouldPublish) { + set({ connectingStep: 'devices' }); applyPublishResult( attempt, await setupPublishPath( @@ -533,40 +656,19 @@ export async function performConnect( state.inputBroadcast, state.pipelineNeedsAudio, state.pipelineNeedsVideo, - set + set, + abortSignal ) ); } - await waitForSignalValue( - attempt.connection.established, - (value) => value !== undefined, - 12_000, - 'Timed out connecting to MoQ gateway.' - ); + await connectWatchPath(attempt, state, decision, set, abortSignal); - if (decision.shouldWatch) { - // When publishing to an external relay, the skit pipeline needs time to - // discover input tracks, build the graph, and start publishing output. - // Wait for the output broadcast to be announced on the relay before - // subscribing, otherwise the catalog subscribe gets RESET_STREAM. - // In gateway mode the skit server manages the peer connection directly, - // so no announcement polling is needed. - if (decision.shouldPublish && state.isExternalRelay) { - await waitForBroadcastAnnouncement(attempt.connection, state.outputBroadcast); - } - - applyWatchResult( - attempt, - setupWatchPath( - attempt.healthEffect, - attempt.connection, - state.outputBroadcast, - state.pipelineOutputsAudio, - state.pipelineOutputsVideo, - set - ) - ); + // If aborted between the last await and now, discard this attempt + // so we don't overwrite a newer connect's state. + if (abortSignal.aborted) { + cleanupConnectAttempt(attempt); + return false; } schedulePostConnectWarnings(decision, attempt, get, set); @@ -574,6 +676,7 @@ export async function performConnect( set({ ...attempt, status: 'connected', + connectingStep: '', isMicEnabled: decision.shouldPublish && state.pipelineNeedsAudio, isCameraEnabled: decision.shouldPublish && state.pipelineNeedsVideo, }); @@ -584,11 +687,18 @@ export async function performConnect( logger.info(`Connection established: ${modes.join(' and ')}`); return true; } catch (error) { - logger.error('Connection failed:', error); cleanupConnectAttempt(attempt); + // If this attempt was aborted (superseded by disconnect or a newer + // connect), silently discard — don't overwrite the store. + if (abortSignal.aborted) { + return false; + } + + logger.error('Connection failed:', error); set({ status: 'disconnected', + connectingStep: '', watchStatus: 'disabled', micStatus: 'disabled', cameraStatus: 'disabled', diff --git a/ui/src/views/StreamView.tsx b/ui/src/views/StreamView.tsx index dc10c804..f7a47106 100644 --- a/ui/src/views/StreamView.tsx +++ b/ui/src/views/StreamView.tsx @@ -269,6 +269,7 @@ const StreamView: React.FC = () => { watchStatus, pipelineNeedsAudio, pipelineNeedsVideo, + connectingStep, errorMessage, configLoaded, activeSessionId, @@ -309,6 +310,7 @@ const StreamView: React.FC = () => { watchStatus: s.watchStatus, pipelineNeedsAudio: s.pipelineNeedsAudio, pipelineNeedsVideo: s.pipelineNeedsVideo, + connectingStep: s.connectingStep, errorMessage: s.errorMessage, configLoaded: s.configLoaded, activeSessionId: s.activeSessionId, @@ -548,9 +550,15 @@ const StreamView: React.FC = () => { if (status === 'disconnected' && serverUrl.trim()) { void (async () => { try { - await connect(); + const ok = await connect(); + if (!ok) { + logger.warn('Auto-connect after session creation did not succeed'); + } } catch (error) { logger.error('MoQ connection attempt after session creation failed:', error); + viewState.setSessionCreationError( + error instanceof Error ? error.message : 'Connection failed after session creation' + ); } })(); } @@ -638,6 +646,12 @@ const StreamView: React.FC = () => { live: 'Watch: live', }; + const connectingStepText: Record = { + devices: 'Requesting device access', + relay: 'Connecting to relay', + pipeline: 'Waiting for pipeline', + }; + return ( { {(status === 'connecting' || status === 'connected') && (
- {status === 'connected' ? 'Relay: connected' : 'Relay: connecting…'} •{' '} - {watchStatusText[watchStatus]} + {status === 'connected' + ? 'Relay: connected' + : connectingStep + ? 'Connecting — ' + (connectingStepText[connectingStep] ?? connectingStep) + : 'Connecting…'}{' '} + • {watchStatusText[watchStatus]} {pipelineNeedsAudio && <> • {micStatusText[micStatus]}} {pipelineNeedsVideo && <> • {cameraStatusText[cameraStatus]}}
From 8e4a83c6d9ec2e3e9aa6184aa0bd5a4a9408ec3f Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 23 Mar 2026 14:42:15 +0000 Subject: [PATCH 19/20] fix: restore gateway URL when switching back from relay pipeline When selecting a MoQ Relay pipeline, the serverUrl in the stream store gets overwritten with the relay URL. Switching back to a Gateway pipeline then applied the gateway path to the relay URL instead of restoring the original config URL, producing an incorrect URL (e.g. http://relay:4443/moq instead of http://127.0.0.1:4545/moq). Fix: add configServerUrl to the stream store that preserves the original URL from /api/v1/config. Extract resolveServerUrl() helper in StreamView that always uses configServerUrl as the base for gateway path resolution. Apply the same fix in useMonitorPreview. Add regression tests: - streamStore: configServerUrl is set by loadConfig and preserved when serverUrl is overwritten - moqPeerSettings: updateUrlPath test suite Co-Authored-By: Claudio Costa --- ui/src/hooks/useMonitorPreview.ts | 9 ++++-- ui/src/stores/streamStore.test.ts | 21 +++++++++++++ ui/src/stores/streamStore.ts | 9 +++++- ui/src/utils/moqPeerSettings.test.ts | 34 +++++++++++++++++++- ui/src/views/StreamView.tsx | 46 +++++++++++++++++----------- 5 files changed, 96 insertions(+), 23 deletions(-) diff --git a/ui/src/hooks/useMonitorPreview.ts b/ui/src/hooks/useMonitorPreview.ts index 29f49765..29631fcc 100644 --- a/ui/src/hooks/useMonitorPreview.ts +++ b/ui/src/hooks/useMonitorPreview.ts @@ -141,9 +141,12 @@ export function useMonitorPreview( } if (gatewayPath) { - const currentUrl = useStreamStore.getState().serverUrl; - if (currentUrl) { - previewSetServerUrl(updateUrlPath(currentUrl, gatewayPath)); + // Use the original config URL as the base so that the preview URL + // isn't polluted by a relay URL the user previously selected. + const baseUrl = + useStreamStore.getState().configServerUrl || useStreamStore.getState().serverUrl; + if (baseUrl) { + previewSetServerUrl(updateUrlPath(baseUrl, gatewayPath)); } } if (outputBroadcast) { diff --git a/ui/src/stores/streamStore.test.ts b/ui/src/stores/streamStore.test.ts index 4a40674c..7d97a405 100644 --- a/ui/src/stores/streamStore.test.ts +++ b/ui/src/stores/streamStore.test.ts @@ -98,6 +98,7 @@ describe('streamStore', () => { pipelineOutputsVideo: true, errorMessage: '', configLoaded: false, + configServerUrl: '', activeSessionId: null, activeSessionName: null, activePipelineName: null, @@ -276,9 +277,29 @@ describe('streamStore', () => { const state = useStreamStore.getState(); expect(state.serverUrl).toBe('http://config-server.com:9000/moq'); + expect(state.configServerUrl).toBe('http://config-server.com:9000/moq'); expect(state.configLoaded).toBe(true); }); + it('should preserve configServerUrl when serverUrl is overwritten by relay', async () => { + const mockConfig = { + moqGatewayUrl: 'http://gateway.example.com:4545/moq', + }; + + vi.mocked(configService.fetchConfig).mockResolvedValue(mockConfig); + + const { loadConfig } = useStreamStore.getState(); + await loadConfig(); + + // Simulate selecting a relay pipeline which overwrites serverUrl + useStreamStore.getState().setServerUrl('http://relay.example.com:4443'); + + const state = useStreamStore.getState(); + expect(state.serverUrl).toBe('http://relay.example.com:4443'); + // configServerUrl must remain the original gateway URL + expect(state.configServerUrl).toBe('http://gateway.example.com:4545/moq'); + }); + it('should set configLoaded when no moqGatewayUrl in config', async () => { const mockConfig = {}; diff --git a/ui/src/stores/streamStore.ts b/ui/src/stores/streamStore.ts index 082268d7..0c502fde 100644 --- a/ui/src/stores/streamStore.ts +++ b/ui/src/stores/streamStore.ts @@ -63,6 +63,8 @@ interface StreamState { // Config state configLoaded: boolean; + /** The original gateway URL from /api/v1/config, preserved across pipeline switches. */ + configServerUrl: string; // Active session state (persisted) activeSessionId: string | null; @@ -148,6 +150,7 @@ export const useStreamStore = create((set, get) => ({ isExternalRelay: false, errorMessage: '', configLoaded: false, + configServerUrl: '', // Active session state activeSessionId: null, @@ -188,7 +191,11 @@ export const useStreamStore = create((set, get) => ({ try { const config = await fetchConfig(); if (config.moqGatewayUrl) { - set({ serverUrl: config.moqGatewayUrl, configLoaded: true }); + set({ + serverUrl: config.moqGatewayUrl, + configServerUrl: config.moqGatewayUrl, + configLoaded: true, + }); } else { set({ configLoaded: true, diff --git a/ui/src/utils/moqPeerSettings.test.ts b/ui/src/utils/moqPeerSettings.test.ts index 0853d430..1091841b 100644 --- a/ui/src/utils/moqPeerSettings.test.ts +++ b/ui/src/utils/moqPeerSettings.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from 'vitest'; -import { extractMoqPeerSettings } from './moqPeerSettings'; +import { extractMoqPeerSettings, updateUrlPath } from './moqPeerSettings'; // --------------------------------------------------------------------------- // extractMoqPeerSettings — reads the declarative `client` section @@ -189,3 +189,35 @@ client: expect(result!.outputsVideo).toBe(false); }); }); + +// --------------------------------------------------------------------------- +// updateUrlPath — preserves host when applying a gateway path +// --------------------------------------------------------------------------- + +describe('updateUrlPath', () => { + it('should replace path on a standard URL', () => { + expect(updateUrlPath('http://127.0.0.1:4545/moq', '/moq/echo')).toBe( + 'http://127.0.0.1:4545/moq/echo' + ); + }); + + it('should replace path on a relay URL (regression: gateway→relay→gateway)', () => { + // If the caller mistakenly passes a relay URL as baseUrl when switching + // back to a gateway pipeline, the result keeps the relay host — which is + // the bug. The fix is for the *caller* to always pass the original + // config URL, but this test documents updateUrlPath's expected behaviour. + expect(updateUrlPath('http://localhost:4443', '/moq')).toBe('http://localhost:4443/moq'); + }); + + it('should handle URLs with trailing slashes', () => { + expect(updateUrlPath('http://example.com:4545/', '/moq/transcoder')).toBe( + 'http://example.com:4545/moq/transcoder' + ); + }); + + it('should preserve protocol and port', () => { + expect(updateUrlPath('https://host.example.com:9443/old-path', '/moq/new')).toBe( + 'https://host.example.com:9443/moq/new' + ); + }); +}); diff --git a/ui/src/views/StreamView.tsx b/ui/src/views/StreamView.tsx index dc10c804..b3efa13b 100644 --- a/ui/src/views/StreamView.tsx +++ b/ui/src/views/StreamView.tsx @@ -31,13 +31,36 @@ import { createSession } from '@/services/sessions'; import { useSchemaStore, ensureSchemasLoaded } from '@/stores/schemaStore'; import type { Event } from '@/types/types'; import { getLogger } from '@/utils/logger'; -import { extractMoqPeerSettings, updateUrlPath } from '@/utils/moqPeerSettings'; +import { + extractMoqPeerSettings, + updateUrlPath, + type MoqPeerSettings, +} from '@/utils/moqPeerSettings'; import { orderSamplePipelinesSystemFirst } from '@/utils/samplePipelineOrdering'; import { useStreamStore } from '../stores/streamStore'; const logger = getLogger('StreamView'); +/** + * Resolves the server URL for a pipeline's MoQ settings. + * + * - Relay pipelines use the relay URL directly. + * - Gateway pipelines apply the gateway path to the original config URL + * (not the current serverUrl, which may have been overwritten by a + * previous relay selection). + * + * Returns the resolved URL or undefined if no update is needed. + */ +function resolveServerUrl(settings: MoqPeerSettings): string | undefined { + if (settings.relayUrl) return settings.relayUrl; + if (settings.gatewayPath) { + const baseUrl = useStreamStore.getState().configServerUrl; + if (baseUrl) return updateUrlPath(baseUrl, settings.gatewayPath); + } + return undefined; +} + const ConnectionControlsRow = styled.div` display: flex; align-items: center; @@ -425,15 +448,8 @@ const StreamView: React.FC = () => { const moqSettings = extractMoqPeerSettings(first.yaml); if (moqSettings) { - if (moqSettings.relayUrl) { - setServerUrl(moqSettings.relayUrl); - } else if (moqSettings.gatewayPath && useStreamStore.getState().serverUrl) { - // Read serverUrl directly from the store since this effect - // runs on mount and the closure-captured value is still ''. - setServerUrl( - updateUrlPath(useStreamStore.getState().serverUrl, moqSettings.gatewayPath) - ); - } + const resolvedUrl = resolveServerUrl(moqSettings); + if (resolvedUrl) setServerUrl(resolvedUrl); if (moqSettings.inputBroadcast) { setInputBroadcast(moqSettings.inputBroadcast); } @@ -471,13 +487,8 @@ const StreamView: React.FC = () => { // Auto-adjust connection settings based on moq_peer node in the pipeline const moqSettings = extractMoqPeerSettings(template.yaml); if (moqSettings) { - // Update server URL: direct relay URL takes priority, otherwise - // apply the gateway path to the current server URL. - if (moqSettings.relayUrl) { - setServerUrl(moqSettings.relayUrl); - } else if (moqSettings.gatewayPath && serverUrl) { - setServerUrl(updateUrlPath(serverUrl, moqSettings.gatewayPath)); - } + const resolvedUrl = resolveServerUrl(moqSettings); + if (resolvedUrl) setServerUrl(resolvedUrl); // Update broadcast names if specified if (moqSettings.inputBroadcast) { setInputBroadcast(moqSettings.inputBroadcast); @@ -506,7 +517,6 @@ const StreamView: React.FC = () => { }, [ viewState, - serverUrl, setServerUrl, setInputBroadcast, setOutputBroadcast, From 88481019ef034d403f1e0dffacba33a951a72150 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 23 Mar 2026 14:53:06 +0000 Subject: [PATCH 20/20] fix: align useMonitorPreview output defaults with deriveSettingsFromClient When client.watch is absent, default outputsAudio/outputsVideo to false (consistent with deriveSettingsFromClient in moqPeerSettings.ts) instead of true. Prevents creating unnecessary audio/video subscribers for pipelines that don't output those media types. Co-Authored-By: Claudio Costa --- ui/src/hooks/useMonitorPreview.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/hooks/useMonitorPreview.ts b/ui/src/hooks/useMonitorPreview.ts index 29631fcc..bfc9e012 100644 --- a/ui/src/hooks/useMonitorPreview.ts +++ b/ui/src/hooks/useMonitorPreview.ts @@ -130,8 +130,8 @@ export function useMonitorPreview( if (client) { gatewayPath = client.gateway_path ?? undefined; outputBroadcast = client.watch?.broadcast; - outputsAudio = client.watch?.audio ?? true; - outputsVideo = client.watch?.video ?? true; + outputsAudio = client.watch?.audio ?? false; + outputsVideo = client.watch?.video ?? false; } else if (pipeline) { const fallback = deriveMoqConfigFromNodes(pipeline); gatewayPath = fallback.gatewayPath;