From 540401558aaf47e9e599e44bbf8dbec7eb3ce93f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Mon, 23 Mar 2026 13:28:41 +0100 Subject: [PATCH 01/18] Add client VAD --- examples/mobile-client/fishjam-chat/app.json | 12 +++- .../fishjam-chat/components/VideosGrid.tsx | 16 +++++- packages/react-client/src/hooks/useVAD.ts | 57 +++++++++++++++---- packages/ts-client/src/FishjamClient.ts | 9 +++ packages/webrtc-client/src/tracks/Local.ts | 13 +++++ .../webrtc-client/src/tracks/LocalTrack.ts | 13 +++++ packages/webrtc-client/src/webRTCEndpoint.ts | 9 +++ 7 files changed, 115 insertions(+), 14 deletions(-) diff --git a/examples/mobile-client/fishjam-chat/app.json b/examples/mobile-client/fishjam-chat/app.json index b44374e14..0b656e63a 100644 --- a/examples/mobile-client/fishjam-chat/app.json +++ b/examples/mobile-client/fishjam-chat/app.json @@ -15,7 +15,13 @@ "NSCameraUsageDescription": "Allow $(PRODUCT_NAME) to access your camera.", "NSMicrophoneUsageDescription": "Allow $(PRODUCT_NAME) to access your microphone.", "ITSAppUsesNonExemptEncryption": false - } + }, + "entitlements": { + "com.apple.security.application-groups": [ + "group.io.fishjam.example.fishjamchat" + ] + }, + "appleTeamId": "J5FM626PE2" }, "android": { "adaptiveIcon": { @@ -71,7 +77,9 @@ } } ], - ["../common/plugins/build/withLocalWebrtcPaths.js"] + [ + "../common/plugins/build/withLocalWebrtcPaths.js" + ] ], "experiments": { "typedRoutes": true diff --git a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx index 82d7b56bb..c8ab9c5f4 100644 --- a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx +++ b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx @@ -1,5 +1,5 @@ -import { RTCView, usePeers } from '@fishjam-cloud/react-native-client'; -import React, { useCallback, useMemo } from 'react'; +import { RTCView, usePeers, useVAD } from '@fishjam-cloud/react-native-client'; +import React, { useCallback, useEffect, useMemo } from 'react'; import type { ListRenderItemInfo } from 'react-native'; import { FlatList, StyleSheet, Text, View } from 'react-native'; @@ -65,6 +65,18 @@ type VideosGridProps = { export default function VideosGrid({ username }: VideosGridProps) { const { localPeer, remotePeers } = usePeers(); const videoTracks = parsePeersToTracks(localPeer, remotePeers); + const allPeerIds = useMemo( + () => [ + ...remotePeers.map((peer) => peer.id), + ...(localPeer ? [localPeer.id] : []), + ], + [remotePeers, localPeer], + ); + const vadStates = useVAD({ peerIds: allPeerIds }); + + useEffect(() => { + console.log('VAD states updated:', vadStates); + }, [vadStates]); const keyExtractor = useCallback( (item: GridTrack, index: number) => item.track?.trackId ?? index.toString(), diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 008e0e123..442491eee 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -1,6 +1,7 @@ import type { TrackContext, VadStatus } from "@fishjam-cloud/ts-client"; import { useContext, useEffect, useMemo, useState } from "react"; +import { FishjamClientContext } from "../contexts/fishjamClient"; import { FishjamClientStateContext } from "../contexts/fishjamState"; import type { PeerId, TrackId } from "../types/public"; @@ -28,16 +29,30 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record - Object.values(clientState.peers) - .filter((peer) => peerIds.includes(peer.id)) - .map((peer) => ({ - peerId: peer.id, - microphoneTracks: Array.from(peer.tracks.values()).filter(({ metadata }) => metadata?.type === "microphone"), - })), - [clientState.peers, peerIds], - ); + const fishjamClient = useContext(FishjamClientContext); + + const micTracksWithSelectedPeerIds = useMemo(() => { + const result = Object.values(clientState.peers) + .filter((peer) => peerIds.includes(peer.id)) + .map((peer) => ({ + peerId: peer.id, + microphoneTracks: Array.from(peer.tracks.values()).filter(({ metadata }) => metadata?.type === "microphone"), + isLocal: false, + })); + + const localPeer = clientState.localPeer; + if (localPeer && peerIds.includes(localPeer.id)) { + result.push({ + peerId: localPeer.id, + microphoneTracks: Array.from(localPeer.tracks.values()).filter( + ({ metadata }) => metadata?.type === "microphone", + ), + isLocal: true, + }); + } + + return result; + }, [clientState.peers, clientState.localPeer, peerIds]); const getDefaultVadStatuses = () => micTracksWithSelectedPeerIds.reduce>>( @@ -76,6 +91,28 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record unsubs.forEach((unsub) => unsub()); }, [micTracksWithSelectedPeerIds]); + useEffect(() => { + const localPeerEntry = micTracksWithSelectedPeerIds.find((e) => e.isLocal); + if (!localPeerEntry || localPeerEntry.microphoneTracks.length === 0) return; + + // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us ~0.025 threshold + const THRESHOLD = 0.025; + const intervals = localPeerEntry.microphoneTracks.map((track) => { + let lastStatus: VadStatus = "silence"; + return setInterval(async () => { + const level = await fishjamClient?.current?.getLocalTrackAudioLevel(track.trackId); + if (level == null) return; + const newStatus: VadStatus = level > THRESHOLD ? "speech" : "silence"; + if (newStatus !== lastStatus) { + lastStatus = newStatus; + fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, newStatus); + } + }, 100); + }); + + return () => intervals.forEach(clearInterval); + }, [micTracksWithSelectedPeerIds, fishjamClient]); + const vadStatuses = useMemo( () => Object.fromEntries( diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index 992bc862f..2cb94d828 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -16,6 +16,7 @@ import type { SimulcastConfig, TrackBandwidthLimit, TrackContext, + VadStatus, Variant, } from '@fishjam-cloud/webrtc-client'; import { getLogger, WebRTCEndpoint } from '@fishjam-cloud/webrtc-client'; @@ -948,4 +949,12 @@ export class FishjamClient { + return this.webrtc?.getLocalTrackAudioLevel(trackId) ?? Promise.resolve(null); + } + + public setLocalTrackVadStatus(trackId: string, vadStatus: VadStatus): void { + this.webrtc?.setLocalTrackVadStatus(trackId, vadStatus); + } } diff --git a/packages/webrtc-client/src/tracks/Local.ts b/packages/webrtc-client/src/tracks/Local.ts index d9ece6bb3..30f58ea5a 100644 --- a/packages/webrtc-client/src/tracks/Local.ts +++ b/packages/webrtc-client/src/tracks/Local.ts @@ -22,6 +22,7 @@ import type { MLineId, RemoteTrackId, TrackBandwidthLimit, + VadStatus, WebRTCEndpointEvents, } from '../types'; import type { WebRTCEndpoint } from '../webRTCEndpoint'; @@ -346,4 +347,16 @@ export class Local { localTrack.addTrackToConnection(); }); }; + + public getLocalTrackAudioLevel = async (trackId: TrackId): Promise => { + return this.localTracks[trackId]?.getAudioLevel() ?? null; + }; + + public setLocalTrackVadStatus = (trackId: TrackId, vadStatus: VadStatus): void => { + const localTrack = this.localTracks[trackId]; + if (!localTrack) return; + if (localTrack.trackContext.vadStatus === vadStatus) return; + localTrack.trackContext.vadStatus = vadStatus; + localTrack.trackContext.emit('voiceActivityChanged', localTrack.trackContext); + }; } diff --git a/packages/webrtc-client/src/tracks/LocalTrack.ts b/packages/webrtc-client/src/tracks/LocalTrack.ts index ee865649c..45861e13c 100644 --- a/packages/webrtc-client/src/tracks/LocalTrack.ts +++ b/packages/webrtc-client/src/tracks/LocalTrack.ts @@ -248,6 +248,19 @@ export class LocalTrack implements TrackCommon { ); }; + public getAudioLevel = async (): Promise => { + if (!this.sender) { + return null; + } + const stats = await this.sender.getStats(); + for (const report of stats.values()) { + if (report.type === 'media-source' && report.kind === 'audio' && typeof report.audioLevel === 'number') { + return report.audioLevel; + } + } + return null; + }; + public createTrackVariantBitratesEvent = () => { // TODO implement this when simulcast is supported // return generateCustomEvent({ diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index b2e9aeb87..7ae44102f 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -33,6 +33,7 @@ import type { DataChannelOptions, TrackBandwidthLimit, TrackContext, + VadStatus, WebRTCEndpointEvents, WebRTCEndpointProps, } from './types'; @@ -119,6 +120,14 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter { + return this.local.getLocalTrackAudioLevel(trackId); + } + + public setLocalTrackVadStatus(trackId: string, vadStatus: VadStatus): void { + this.local.setLocalTrackVadStatus(trackId, vadStatus); + } + /** * Feeds media event received from RTC Engine to {@link WebRTCEndpoint}. * This function should be called whenever some media event from RTC Engine From e0199ee19e98f76a4bf0f12a521c1b45b566ff27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Mon, 23 Mar 2026 19:26:57 +0100 Subject: [PATCH 02/18] Change api for client vad --- packages/react-client/src/hooks/useVAD.ts | 79 ++++++++++--------- packages/ts-client/src/FishjamClient.ts | 2 +- packages/webrtc-client/src/tracks/Local.ts | 2 +- .../webrtc-client/src/tracks/LocalTrack.ts | 5 +- packages/webrtc-client/src/webRTCEndpoint.ts | 2 +- 5 files changed, 48 insertions(+), 42 deletions(-) diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 442491eee..c0cd9bd2d 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -24,8 +24,11 @@ import type { PeerId, TrackId } from "../types/public"; * @group Hooks * @returns Each key is a peerId and the boolean value indicates if voice activity is currently detected for that peer. */ -export const useVAD = (options: { peerIds: ReadonlyArray }): Record => { - const { peerIds } = options; +export const useVAD = (options: { + peerIds: ReadonlyArray; + showLocalPeer?: boolean; +}): Record => { + const { peerIds, showLocalPeer } = options; const clientState = useContext(FishjamClientStateContext); if (!clientState) throw Error("useVAD must be used within FishjamProvider"); @@ -36,32 +39,29 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record peerIds.includes(peer.id)) .map((peer) => ({ peerId: peer.id, - microphoneTracks: Array.from(peer.tracks.values()).filter(({ metadata }) => metadata?.type === "microphone"), + microphoneTrack: Array.from(peer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone"), isLocal: false, })); const localPeer = clientState.localPeer; - if (localPeer && peerIds.includes(localPeer.id)) { + if (localPeer && showLocalPeer) { result.push({ peerId: localPeer.id, - microphoneTracks: Array.from(localPeer.tracks.values()).filter( - ({ metadata }) => metadata?.type === "microphone", - ), + microphoneTrack: Array.from(localPeer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone"), isLocal: true, }); } return result; - }, [clientState.peers, clientState.localPeer, peerIds]); + }, [clientState.peers, clientState.localPeer, peerIds, showLocalPeer]); const getDefaultVadStatuses = () => micTracksWithSelectedPeerIds.reduce>>( - (mappedTracks, peer) => ({ + (mappedTracks, { peerId, microphoneTrack }) => ({ ...mappedTracks, - [peer.peerId]: peer.microphoneTracks.reduce( - (vadStatuses, track) => ({ ...vadStatuses, [track.trackId]: track.vadStatus }), - {}, - ), + [peerId]: microphoneTrack + ? { [(microphoneTrack as TrackContext).trackId]: (microphoneTrack as TrackContext).vadStatus } + : {}, }), {}, ); @@ -69,7 +69,7 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record>>(getDefaultVadStatuses); useEffect(() => { - const unsubs = micTracksWithSelectedPeerIds.map(({ peerId, microphoneTracks }) => { + const unsubs = micTracksWithSelectedPeerIds.map(({ peerId, microphoneTrack }) => { const updateVadStatus = (track: TrackContext) => { setVadStatuses((prev) => ({ ...prev, @@ -77,14 +77,14 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record { - track.on("voiceActivityChanged", updateVadStatus); - }); + if (microphoneTrack) { + (microphoneTrack as TrackContext).on("voiceActivityChanged", updateVadStatus); + } return () => { - microphoneTracks.forEach((track) => { - track.off("voiceActivityChanged", updateVadStatus); - }); + if (microphoneTrack) { + (microphoneTrack as TrackContext).off("voiceActivityChanged", updateVadStatus); + } }; }); @@ -92,26 +92,31 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record { + if (!showLocalPeer) return; const localPeerEntry = micTracksWithSelectedPeerIds.find((e) => e.isLocal); - if (!localPeerEntry || localPeerEntry.microphoneTracks.length === 0) return; - + if (!localPeerEntry || !localPeerEntry.microphoneTrack) return; // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us ~0.025 threshold const THRESHOLD = 0.025; - const intervals = localPeerEntry.microphoneTracks.map((track) => { - let lastStatus: VadStatus = "silence"; - return setInterval(async () => { - const level = await fishjamClient?.current?.getLocalTrackAudioLevel(track.trackId); - if (level == null) return; - const newStatus: VadStatus = level > THRESHOLD ? "speech" : "silence"; - if (newStatus !== lastStatus) { - lastStatus = newStatus; - fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, newStatus); - } - }, 100); - }); - - return () => intervals.forEach(clearInterval); - }, [micTracksWithSelectedPeerIds, fishjamClient]); + const track = localPeerEntry.microphoneTrack as TrackContext; + let lastStatus: VadStatus = track.vadStatus; + const interval = setInterval(async () => { + const level = await fishjamClient?.current?.getLocalTrackAudioLevel(track.trackId); + if (level == null) return; + // console.log( + // `[VAD] Local track audio level for track ${track.trackId}:`, + // level.level, + // "timestamp:", + // level.timestamp, + // ); + const newStatus: VadStatus = level.level > THRESHOLD ? "speech" : "silence"; + if (newStatus !== lastStatus) { + lastStatus = newStatus; + fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, newStatus); + } + }, 100); + + return () => clearInterval(interval); + }, [showLocalPeer, micTracksWithSelectedPeerIds, fishjamClient]); const vadStatuses = useMemo( () => diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index 2cb94d828..e88680891 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -950,7 +950,7 @@ export class FishjamClient { + public getLocalTrackAudioLevel(trackId: string): Promise<{ level: number; timestamp: number } | null> { return this.webrtc?.getLocalTrackAudioLevel(trackId) ?? Promise.resolve(null); } diff --git a/packages/webrtc-client/src/tracks/Local.ts b/packages/webrtc-client/src/tracks/Local.ts index 30f58ea5a..72f8bbb2a 100644 --- a/packages/webrtc-client/src/tracks/Local.ts +++ b/packages/webrtc-client/src/tracks/Local.ts @@ -348,7 +348,7 @@ export class Local { }); }; - public getLocalTrackAudioLevel = async (trackId: TrackId): Promise => { + public getLocalTrackAudioLevel = async (trackId: TrackId): Promise<{ level: number; timestamp: number } | null> => { return this.localTracks[trackId]?.getAudioLevel() ?? null; }; diff --git a/packages/webrtc-client/src/tracks/LocalTrack.ts b/packages/webrtc-client/src/tracks/LocalTrack.ts index 45861e13c..02643a16b 100644 --- a/packages/webrtc-client/src/tracks/LocalTrack.ts +++ b/packages/webrtc-client/src/tracks/LocalTrack.ts @@ -248,14 +248,15 @@ export class LocalTrack implements TrackCommon { ); }; - public getAudioLevel = async (): Promise => { + public getAudioLevel = async (): Promise<{ level: number; timestamp: number } | null> => { if (!this.sender) { return null; } + const stats = await this.sender.getStats(); for (const report of stats.values()) { if (report.type === 'media-source' && report.kind === 'audio' && typeof report.audioLevel === 'number') { - return report.audioLevel; + return { level: report.audioLevel, timestamp: report.timestamp }; } } return null; diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index 7ae44102f..a194c7b26 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -120,7 +120,7 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter { + public getLocalTrackAudioLevel(trackId: string): Promise<{ level: number; timestamp: number } | null> { return this.local.getLocalTrackAudioLevel(trackId); } From d8823876b17bf3301127d31e3a2dcd7e3229c128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Mon, 23 Mar 2026 19:46:53 +0100 Subject: [PATCH 03/18] Remove timestamp --- packages/react-client/src/hooks/useVAD.ts | 25 +++++++++++-------- packages/ts-client/src/FishjamClient.ts | 2 +- packages/webrtc-client/src/tracks/Local.ts | 2 +- .../webrtc-client/src/tracks/LocalTrack.ts | 4 +-- packages/webrtc-client/src/webRTCEndpoint.ts | 2 +- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index c0cd9bd2d..49fa7f76a 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -97,21 +97,26 @@ export const useVAD = (options: { if (!localPeerEntry || !localPeerEntry.microphoneTrack) return; // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us ~0.025 threshold const THRESHOLD = 0.025; + const SILENCE_DEBOUNCE_TICKS = 2; const track = localPeerEntry.microphoneTrack as TrackContext; let lastStatus: VadStatus = track.vadStatus; + let silenceTicks = 0; const interval = setInterval(async () => { const level = await fishjamClient?.current?.getLocalTrackAudioLevel(track.trackId); if (level == null) return; - // console.log( - // `[VAD] Local track audio level for track ${track.trackId}:`, - // level.level, - // "timestamp:", - // level.timestamp, - // ); - const newStatus: VadStatus = level.level > THRESHOLD ? "speech" : "silence"; - if (newStatus !== lastStatus) { - lastStatus = newStatus; - fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, newStatus); + const isSpeech = level.level > THRESHOLD; + if (isSpeech) { + silenceTicks = 0; + if (lastStatus !== "speech") { + lastStatus = "speech"; + fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, "speech"); + } + } else { + silenceTicks += 1; + if (silenceTicks >= SILENCE_DEBOUNCE_TICKS && lastStatus !== "silence") { + lastStatus = "silence"; + fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, "silence"); + } } }, 100); diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index e88680891..d747fc477 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -950,7 +950,7 @@ export class FishjamClient { + public getLocalTrackAudioLevel(trackId: string): Promise<{ level: number } | null> { return this.webrtc?.getLocalTrackAudioLevel(trackId) ?? Promise.resolve(null); } diff --git a/packages/webrtc-client/src/tracks/Local.ts b/packages/webrtc-client/src/tracks/Local.ts index 72f8bbb2a..ea09adf78 100644 --- a/packages/webrtc-client/src/tracks/Local.ts +++ b/packages/webrtc-client/src/tracks/Local.ts @@ -348,7 +348,7 @@ export class Local { }); }; - public getLocalTrackAudioLevel = async (trackId: TrackId): Promise<{ level: number; timestamp: number } | null> => { + public getLocalTrackAudioLevel = async (trackId: TrackId): Promise<{ level: number } | null> => { return this.localTracks[trackId]?.getAudioLevel() ?? null; }; diff --git a/packages/webrtc-client/src/tracks/LocalTrack.ts b/packages/webrtc-client/src/tracks/LocalTrack.ts index 02643a16b..f5f90d448 100644 --- a/packages/webrtc-client/src/tracks/LocalTrack.ts +++ b/packages/webrtc-client/src/tracks/LocalTrack.ts @@ -248,7 +248,7 @@ export class LocalTrack implements TrackCommon { ); }; - public getAudioLevel = async (): Promise<{ level: number; timestamp: number } | null> => { + public getAudioLevel = async (): Promise<{ level: number } | null> => { if (!this.sender) { return null; } @@ -256,7 +256,7 @@ export class LocalTrack implements TrackCommon { const stats = await this.sender.getStats(); for (const report of stats.values()) { if (report.type === 'media-source' && report.kind === 'audio' && typeof report.audioLevel === 'number') { - return { level: report.audioLevel, timestamp: report.timestamp }; + return { level: report.audioLevel }; } } return null; diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index a194c7b26..e5018311c 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -120,7 +120,7 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter { + public getLocalTrackAudioLevel(trackId: string): Promise<{ level: number } | null> { return this.local.getLocalTrackAudioLevel(trackId); } From 3ced8eb7084928a8cdb8268ce80c93213414defd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 13:07:54 +0100 Subject: [PATCH 04/18] Add useLocalVAD and simplify the logic --- .../react-client/src/hooks/useLocalVAD.ts | 53 ++++++++++++ packages/react-client/src/hooks/useVAD.ts | 83 +++++-------------- 2 files changed, 75 insertions(+), 61 deletions(-) create mode 100644 packages/react-client/src/hooks/useLocalVAD.ts diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts new file mode 100644 index 000000000..677b9318c --- /dev/null +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -0,0 +1,53 @@ +import type { GenericMetadata } from "@fishjam-cloud/ts-client"; +import { useContext, useEffect, useState } from "react"; + +import { FishjamClientContext } from "../contexts/fishjamClient"; +import type { BrandedPeer } from "../types/internal"; +import type { PeerId } from "../types/public"; + +export const useLocalVAD = ( + localPeer: BrandedPeer | null, + showLocalPeer?: boolean, +): Record => { + const fishjamClient = useContext(FishjamClientContext); + const [isSpeaking, setIsSpeaking] = useState(false); + + const microphoneTrack = localPeer + ? Array.from(localPeer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone") + : undefined; + + useEffect(() => { + if (!showLocalPeer || !localPeer) return; + if (!microphoneTrack) return; + // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us ~0.025 threshold + const THRESHOLD = 0.025; + const SILENCE_DEBOUNCE_TICKS = 2; + let isSpeech = false; + let silenceTicks = 0; + const interval = setInterval(async () => { + const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrack.trackId); + if (trackAudio == null) return; + if (trackAudio.level > THRESHOLD) { + silenceTicks = 0; + if (!isSpeech) { + isSpeech = true; + setIsSpeaking(true); + } + } else { + silenceTicks += 1; + if (silenceTicks >= SILENCE_DEBOUNCE_TICKS && isSpeech) { + isSpeech = false; + setIsSpeaking(false); + } + } + }, 100); + + return () => { + clearInterval(interval); + setIsSpeaking(false); + }; + }, [showLocalPeer, fishjamClient, microphoneTrack, localPeer]); + + if (!localPeer || !showLocalPeer) return {}; + return { [localPeer.id]: isSpeaking }; +}; diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 49fa7f76a..3f85d7080 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -1,9 +1,9 @@ import type { TrackContext, VadStatus } from "@fishjam-cloud/ts-client"; import { useContext, useEffect, useMemo, useState } from "react"; -import { FishjamClientContext } from "../contexts/fishjamClient"; import { FishjamClientStateContext } from "../contexts/fishjamState"; import type { PeerId, TrackId } from "../types/public"; +import { useLocalVAD } from "./useLocalVAD"; /** * Voice activity detection. Use this hook to check if voice is detected in audio track for given peer(s). @@ -32,28 +32,16 @@ export const useVAD = (options: { const clientState = useContext(FishjamClientStateContext); if (!clientState) throw Error("useVAD must be used within FishjamProvider"); - const fishjamClient = useContext(FishjamClientContext); - - const micTracksWithSelectedPeerIds = useMemo(() => { - const result = Object.values(clientState.peers) - .filter((peer) => peerIds.includes(peer.id)) - .map((peer) => ({ - peerId: peer.id, - microphoneTrack: Array.from(peer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone"), - isLocal: false, - })); - - const localPeer = clientState.localPeer; - if (localPeer && showLocalPeer) { - result.push({ - peerId: localPeer.id, - microphoneTrack: Array.from(localPeer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone"), - isLocal: true, - }); - } - - return result; - }, [clientState.peers, clientState.localPeer, peerIds, showLocalPeer]); + const micTracksWithSelectedPeerIds = useMemo( + () => + Object.values(clientState.peers) + .filter((peer) => peerIds.includes(peer.id)) + .map((peer) => ({ + peerId: peer.id, + microphoneTrack: Array.from(peer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone"), + })), + [clientState.peers, peerIds], + ); const getDefaultVadStatuses = () => micTracksWithSelectedPeerIds.reduce>>( @@ -91,47 +79,20 @@ export const useVAD = (options: { return () => unsubs.forEach((unsub) => unsub()); }, [micTracksWithSelectedPeerIds]); - useEffect(() => { - if (!showLocalPeer) return; - const localPeerEntry = micTracksWithSelectedPeerIds.find((e) => e.isLocal); - if (!localPeerEntry || !localPeerEntry.microphoneTrack) return; - // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us ~0.025 threshold - const THRESHOLD = 0.025; - const SILENCE_DEBOUNCE_TICKS = 2; - const track = localPeerEntry.microphoneTrack as TrackContext; - let lastStatus: VadStatus = track.vadStatus; - let silenceTicks = 0; - const interval = setInterval(async () => { - const level = await fishjamClient?.current?.getLocalTrackAudioLevel(track.trackId); - if (level == null) return; - const isSpeech = level.level > THRESHOLD; - if (isSpeech) { - silenceTicks = 0; - if (lastStatus !== "speech") { - lastStatus = "speech"; - fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, "speech"); - } - } else { - silenceTicks += 1; - if (silenceTicks >= SILENCE_DEBOUNCE_TICKS && lastStatus !== "silence") { - lastStatus = "silence"; - fishjamClient?.current?.setLocalTrackVadStatus(track.trackId, "silence"); - } - } - }, 100); - - return () => clearInterval(interval); - }, [showLocalPeer, micTracksWithSelectedPeerIds, fishjamClient]); + const localVAD = useLocalVAD(clientState.localPeer, showLocalPeer); const vadStatuses = useMemo( () => - Object.fromEntries( - Object.entries(_vadStatuses).map(([peerId, tracks]) => [ - peerId, - Object.values(tracks).some((vad) => vad === "speech"), - ]), - ) satisfies Record, - [_vadStatuses], + ({ + ...Object.fromEntries( + Object.entries(_vadStatuses).map(([peerId, tracks]) => [ + peerId, + Object.values(tracks).some((vad) => vad === "speech"), + ]), + ), + ...localVAD, + }) satisfies Record, + [_vadStatuses, localVAD], ); return vadStatuses; From 50da0ebed7ae02f06fc15e9a3110107a89499aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 15:06:02 +0100 Subject: [PATCH 05/18] Remove setLocalVadStatus --- packages/ts-client/src/FishjamClient.ts | 5 ----- packages/webrtc-client/src/tracks/Local.ts | 9 --------- packages/webrtc-client/src/webRTCEndpoint.ts | 5 ----- 3 files changed, 19 deletions(-) diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index d747fc477..09d8b96e9 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -16,7 +16,6 @@ import type { SimulcastConfig, TrackBandwidthLimit, TrackContext, - VadStatus, Variant, } from '@fishjam-cloud/webrtc-client'; import { getLogger, WebRTCEndpoint } from '@fishjam-cloud/webrtc-client'; @@ -953,8 +952,4 @@ export class FishjamClient { return this.webrtc?.getLocalTrackAudioLevel(trackId) ?? Promise.resolve(null); } - - public setLocalTrackVadStatus(trackId: string, vadStatus: VadStatus): void { - this.webrtc?.setLocalTrackVadStatus(trackId, vadStatus); - } } diff --git a/packages/webrtc-client/src/tracks/Local.ts b/packages/webrtc-client/src/tracks/Local.ts index ea09adf78..83a4b488c 100644 --- a/packages/webrtc-client/src/tracks/Local.ts +++ b/packages/webrtc-client/src/tracks/Local.ts @@ -22,7 +22,6 @@ import type { MLineId, RemoteTrackId, TrackBandwidthLimit, - VadStatus, WebRTCEndpointEvents, } from '../types'; import type { WebRTCEndpoint } from '../webRTCEndpoint'; @@ -351,12 +350,4 @@ export class Local { public getLocalTrackAudioLevel = async (trackId: TrackId): Promise<{ level: number } | null> => { return this.localTracks[trackId]?.getAudioLevel() ?? null; }; - - public setLocalTrackVadStatus = (trackId: TrackId, vadStatus: VadStatus): void => { - const localTrack = this.localTracks[trackId]; - if (!localTrack) return; - if (localTrack.trackContext.vadStatus === vadStatus) return; - localTrack.trackContext.vadStatus = vadStatus; - localTrack.trackContext.emit('voiceActivityChanged', localTrack.trackContext); - }; } diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index e5018311c..25477a172 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -33,7 +33,6 @@ import type { DataChannelOptions, TrackBandwidthLimit, TrackContext, - VadStatus, WebRTCEndpointEvents, WebRTCEndpointProps, } from './types'; @@ -124,10 +123,6 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter Date: Tue, 24 Mar 2026 15:09:42 +0100 Subject: [PATCH 06/18] Change api for useLocalVAD and add/change JSDocs --- .../react-client/src/hooks/useLocalVAD.ts | 43 +++++++++++-------- packages/react-client/src/hooks/useVAD.ts | 26 ++++++----- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index 677b9318c..af1073116 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -1,31 +1,40 @@ -import type { GenericMetadata } from "@fishjam-cloud/ts-client"; import { useContext, useEffect, useState } from "react"; import { FishjamClientContext } from "../contexts/fishjamClient"; -import type { BrandedPeer } from "../types/internal"; import type { PeerId } from "../types/public"; +import { usePeers } from "./usePeers"; -export const useLocalVAD = ( - localPeer: BrandedPeer | null, - showLocalPeer?: boolean, -): Record => { +/** + * Client-side voice activity detection for the local peer. + * + * Polls the local microphone's audio level every 100ms and derives a speech/silence + * state from it. A level above ~0.025 (approximately −32 dBov, scaled to [0, 1]) + * is treated as speech. Silence is debounced over 2 consecutive ticks (~200ms) + * to prevent rapid flapping. + * + * This is purely client-side — it does not signal other peers. Remote participants + * receive the local peer's VAD status via backend `vadNotification` messages. + * + * @internal Used by `useVAD` when the local peer's id is included in `peerIds`. + * @returns A record mapping the local peer's id to its current speaking state, + * or an empty object if `showLocalPeer` is false or no microphone track is found. + */ +export const useLocalVAD = (showLocalPeer: boolean): Record => { const fishjamClient = useContext(FishjamClientContext); const [isSpeaking, setIsSpeaking] = useState(false); - - const microphoneTrack = localPeer - ? Array.from(localPeer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone") - : undefined; + const { localPeer } = usePeers(); + const localPeerId = localPeer?.id; + const microphoneTrackId = localPeer?.microphoneTrack?.trackId; useEffect(() => { - if (!showLocalPeer || !localPeer) return; - if (!microphoneTrack) return; - // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us ~0.025 threshold + if (!showLocalPeer || !localPeerId || !microphoneTrackId) return; + // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us 0.025 threshold const THRESHOLD = 0.025; const SILENCE_DEBOUNCE_TICKS = 2; let isSpeech = false; let silenceTicks = 0; const interval = setInterval(async () => { - const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrack.trackId); + const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrackId); if (trackAudio == null) return; if (trackAudio.level > THRESHOLD) { silenceTicks = 0; @@ -46,8 +55,8 @@ export const useLocalVAD = ( clearInterval(interval); setIsSpeaking(false); }; - }, [showLocalPeer, fishjamClient, microphoneTrack, localPeer]); + }, [showLocalPeer, fishjamClient, localPeerId, microphoneTrackId]); - if (!localPeer || !showLocalPeer) return {}; - return { [localPeer.id]: isSpeaking }; + if (!localPeerId || !showLocalPeer) return {}; + return { [localPeerId]: isSpeaking }; }; diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 3f85d7080..7f7ca74ad 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -6,15 +6,22 @@ import type { PeerId, TrackId } from "../types/public"; import { useLocalVAD } from "./useLocalVAD"; /** - * Voice activity detection. Use this hook to check if voice is detected in audio track for given peer(s). + * Voice activity detection. Use this hook to check if voice is detected in the audio track for given peer(s). * - * @param options - Options object containing `peerIds` - a list of ids of peers to subscribe to for voice activity detection notifications. + * Remote peer VAD is driven by `vadNotification` messages from the backend. + * If the local peer's id is included in `peerIds`, local VAD is determined client-side + * by polling the microphone's audio level (see `useLocalVAD`). + * + * @param options - Options object. + * @param options.peerIds - List of peer ids to subscribe to for VAD notifications. + * Include the local peer's id to also track whether the local user is speaking. * * Example usage: * ```tsx * import { useVAD, type PeerId } from "@fishjam-cloud/react-client"; + * * function WhoIsTalkingComponent({ peerIds }: { peerIds: PeerId[] }) { - * const peersInfo = useVAD({peerIds}); + * const peersInfo = useVAD({ peerIds }); * const activePeers = (Object.keys(peersInfo) as PeerId[]).filter((peerId) => peersInfo[peerId]); * * return "Now talking: " + activePeers.join(", "); @@ -22,15 +29,14 @@ import { useLocalVAD } from "./useLocalVAD"; * ``` * @category Connection * @group Hooks - * @returns Each key is a peerId and the boolean value indicates if voice activity is currently detected for that peer. + * @returns A record where each key is a peer id and the boolean value indicates + * whether voice activity is currently detected for that peer. */ -export const useVAD = (options: { - peerIds: ReadonlyArray; - showLocalPeer?: boolean; -}): Record => { - const { peerIds, showLocalPeer } = options; +export const useVAD = (options: { peerIds: ReadonlyArray }): Record => { + const { peerIds } = options; const clientState = useContext(FishjamClientStateContext); if (!clientState) throw Error("useVAD must be used within FishjamProvider"); + const showLocalPeer = clientState.localPeer?.id ? peerIds.includes(clientState.localPeer?.id) : false; const micTracksWithSelectedPeerIds = useMemo( () => @@ -79,7 +85,7 @@ export const useVAD = (options: { return () => unsubs.forEach((unsub) => unsub()); }, [micTracksWithSelectedPeerIds]); - const localVAD = useLocalVAD(clientState.localPeer, showLocalPeer); + const localVAD = useLocalVAD(showLocalPeer); const vadStatuses = useMemo( () => From 8cee0adfa5fa8edab60e70965e657cd4434c9486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 15:20:53 +0100 Subject: [PATCH 07/18] Add dynamic scaling of dbOv to linear scale --- packages/react-client/src/hooks/useLocalVAD.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index af1073116..9e18906f5 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -28,8 +28,8 @@ export const useLocalVAD = (showLocalPeer: boolean): Record => useEffect(() => { if (!showLocalPeer || !localPeerId || !microphoneTrackId) return; - // above -32 dBov -> speech, below -> silence, scaled to [0, 1] range gives us 0.025 threshold - const THRESHOLD = 0.025; + // above -32 dBov -> speech, below -> silence; convert dBov to linear [0, 1] scale: + const THRESHOLD = 10 ** (-32 / 20); const SILENCE_DEBOUNCE_TICKS = 2; let isSpeech = false; let silenceTicks = 0; From d57b8321f800298d47d785e2002a6b435bbe45b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 15:23:37 +0100 Subject: [PATCH 08/18] Add cue showing that peer is speaking --- .../fishjam-chat/components/VideosGrid.tsx | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx index c8ab9c5f4..5dd66d260 100644 --- a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx +++ b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx @@ -1,5 +1,6 @@ +import type { PeerId } from '@fishjam-cloud/react-native-client'; import { RTCView, usePeers, useVAD } from '@fishjam-cloud/react-native-client'; -import React, { useCallback, useEffect, useMemo } from 'react'; +import React, { useCallback, useMemo } from 'react'; import type { ListRenderItemInfo } from 'react-native'; import { FlatList, StyleSheet, Text, View } from 'react-native'; @@ -11,9 +12,11 @@ import NoCameraView from './NoCameraView'; const GridTrackItem = ({ peer, _index, + isSpeaking, }: { peer: GridTrack; _index: number; + isSpeaking: boolean; }) => { const isSelfVideo = peer.isLocal && peer.track?.metadata?.type === 'camera'; const isCamera = peer.track?.metadata?.type === 'camera'; @@ -31,6 +34,8 @@ const GridTrackItem = ({ backgroundColor: peer.isLocal ? BrandColors.seaBlue60 : BrandColors.darkBlue60, + borderColor: isSpeaking ? '#00ff1a' : BrandColors.darkBlue100, + borderWidth: 2, }, ]}> {mediaStream ? ( @@ -65,18 +70,13 @@ type VideosGridProps = { export default function VideosGrid({ username }: VideosGridProps) { const { localPeer, remotePeers } = usePeers(); const videoTracks = parsePeersToTracks(localPeer, remotePeers); - const allPeerIds = useMemo( - () => [ - ...remotePeers.map((peer) => peer.id), - ...(localPeer ? [localPeer.id] : []), - ], - [remotePeers, localPeer], - ); - const vadStates = useVAD({ peerIds: allPeerIds }); - - useEffect(() => { - console.log('VAD states updated:', vadStates); - }, [vadStates]); + const vadPeers = useMemo(() => { + const remoteIds = remotePeers.map((peer) => peer.id);; + return localPeer?.id ? [localPeer.id, ...remoteIds] : remoteIds; + }, [remotePeers, localPeer]); + const vadStates = useVAD({ + peerIds: vadPeers, + }); const keyExtractor = useCallback( (item: GridTrack, index: number) => item.track?.trackId ?? index.toString(), @@ -85,9 +85,16 @@ export default function VideosGrid({ username }: VideosGridProps) { const renderItem = useCallback( ({ item, index }: ListRenderItemInfo) => ( - + ), - [], + [vadStates], ); const ListEmptyComponent = useMemo( From 7a0bc5d7174917506ec3faf6038a07e818af806a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 16:48:22 +0100 Subject: [PATCH 09/18] Fix issues outlined by copilot --- examples/mobile-client/fishjam-chat/app.json | 8 +- .../fishjam-chat/components/VideosGrid.tsx | 4 +- .../fishjam-chat/utils/tracks.ts | 4 +- .../react-client/src/hooks/useLocalVAD.ts | 44 +++++++---- packages/react-client/src/hooks/useVAD.ts | 12 ++- .../getLocalTrackAudioLevelMethod.test.ts | 73 +++++++++++++++++++ 6 files changed, 112 insertions(+), 33 deletions(-) create mode 100644 packages/webrtc-client/tests/methods/getLocalTrackAudioLevelMethod.test.ts diff --git a/examples/mobile-client/fishjam-chat/app.json b/examples/mobile-client/fishjam-chat/app.json index 0b656e63a..a336812d3 100644 --- a/examples/mobile-client/fishjam-chat/app.json +++ b/examples/mobile-client/fishjam-chat/app.json @@ -15,13 +15,7 @@ "NSCameraUsageDescription": "Allow $(PRODUCT_NAME) to access your camera.", "NSMicrophoneUsageDescription": "Allow $(PRODUCT_NAME) to access your microphone.", "ITSAppUsesNonExemptEncryption": false - }, - "entitlements": { - "com.apple.security.application-groups": [ - "group.io.fishjam.example.fishjamchat" - ] - }, - "appleTeamId": "J5FM626PE2" + } }, "android": { "adaptiveIcon": { diff --git a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx index 5dd66d260..6053feb96 100644 --- a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx +++ b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx @@ -71,7 +71,7 @@ export default function VideosGrid({ username }: VideosGridProps) { const { localPeer, remotePeers } = usePeers(); const videoTracks = parsePeersToTracks(localPeer, remotePeers); const vadPeers = useMemo(() => { - const remoteIds = remotePeers.map((peer) => peer.id);; + const remoteIds = remotePeers.map((peer) => peer.id); return localPeer?.id ? [localPeer.id, ...remoteIds] : remoteIds; }, [remotePeers, localPeer]); const vadStates = useVAD({ @@ -89,7 +89,7 @@ export default function VideosGrid({ username }: VideosGridProps) { peer={item} _index={index} isSpeaking={ - !!vadStates[item.peerId as PeerId] && + !!vadStates[item.peerId] && item.track?.metadata?.type !== 'screenShareVideo' } /> diff --git a/examples/mobile-client/fishjam-chat/utils/tracks.ts b/examples/mobile-client/fishjam-chat/utils/tracks.ts index 5bff3937f..762da9270 100644 --- a/examples/mobile-client/fishjam-chat/utils/tracks.ts +++ b/examples/mobile-client/fishjam-chat/utils/tracks.ts @@ -1,8 +1,8 @@ -import type { PeerWithTracks, Track } from '@fishjam-cloud/react-native-client'; +import type { PeerId, PeerWithTracks, Track } from '@fishjam-cloud/react-native-client'; export type GridTrack = { track: Track | null; - peerId: string; + peerId: PeerId; isLocal: boolean; isVadActive: boolean; aspectRatio: number | null; diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index 9e18906f5..e09e47d52 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -31,32 +31,46 @@ export const useLocalVAD = (showLocalPeer: boolean): Record => // above -32 dBov -> speech, below -> silence; convert dBov to linear [0, 1] scale: const THRESHOLD = 10 ** (-32 / 20); const SILENCE_DEBOUNCE_TICKS = 2; - let isSpeech = false; let silenceTicks = 0; + let isSpeech = false; + let cancelled = false; + let polling = false; const interval = setInterval(async () => { - const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrackId); - if (trackAudio == null) return; - if (trackAudio.level > THRESHOLD) { - silenceTicks = 0; - if (!isSpeech) { - isSpeech = true; - setIsSpeaking(true); - } - } else { - silenceTicks += 1; - if (silenceTicks >= SILENCE_DEBOUNCE_TICKS && isSpeech) { - isSpeech = false; - setIsSpeaking(false); + if (cancelled || polling) return; + polling = true; + try { + const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrackId); + if (cancelled || trackAudio == null) return; + if (trackAudio.level > THRESHOLD) { + silenceTicks = 0; + if (!isSpeech) { + isSpeech = true; + setIsSpeaking(true); + } + } else { + silenceTicks += 1; + if (silenceTicks >= SILENCE_DEBOUNCE_TICKS && isSpeech) { + isSpeech = false; + setIsSpeaking(false); + } } + } catch { + console.error("Error polling local track audio level for VAD", { + peerId: localPeerId, + trackId: microphoneTrackId, + }); + } finally { + polling = false; } }, 100); return () => { + cancelled = true; clearInterval(interval); setIsSpeaking(false); }; }, [showLocalPeer, fishjamClient, localPeerId, microphoneTrackId]); - if (!localPeerId || !showLocalPeer) return {}; + if (!localPeerId || !showLocalPeer || !microphoneTrackId) return {}; return { [localPeerId]: isSpeaking }; }; diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 7f7ca74ad..12a953315 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -1,4 +1,4 @@ -import type { TrackContext, VadStatus } from "@fishjam-cloud/ts-client"; +import type { FishjamTrackContext, VadStatus } from "@fishjam-cloud/ts-client"; import { useContext, useEffect, useMemo, useState } from "react"; import { FishjamClientStateContext } from "../contexts/fishjamState"; @@ -53,9 +53,7 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record>>( (mappedTracks, { peerId, microphoneTrack }) => ({ ...mappedTracks, - [peerId]: microphoneTrack - ? { [(microphoneTrack as TrackContext).trackId]: (microphoneTrack as TrackContext).vadStatus } - : {}, + [peerId]: microphoneTrack ? { [microphoneTrack.trackId]: microphoneTrack.vadStatus } : {}, }), {}, ); @@ -64,7 +62,7 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record { const unsubs = micTracksWithSelectedPeerIds.map(({ peerId, microphoneTrack }) => { - const updateVadStatus = (track: TrackContext) => { + const updateVadStatus = (track: FishjamTrackContext) => { setVadStatuses((prev) => ({ ...prev, [peerId]: { ...prev[peerId], [track.trackId]: track.vadStatus }, @@ -72,12 +70,12 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record { if (microphoneTrack) { - (microphoneTrack as TrackContext).off("voiceActivityChanged", updateVadStatus); + microphoneTrack.off("voiceActivityChanged", updateVadStatus); } }; }); diff --git a/packages/webrtc-client/tests/methods/getLocalTrackAudioLevelMethod.test.ts b/packages/webrtc-client/tests/methods/getLocalTrackAudioLevelMethod.test.ts new file mode 100644 index 000000000..4844753c0 --- /dev/null +++ b/packages/webrtc-client/tests/methods/getLocalTrackAudioLevelMethod.test.ts @@ -0,0 +1,73 @@ +import { expect, it, vi } from 'vitest'; + +import { WebRTCEndpoint } from '../../src'; +import { serializeServerMediaEvent } from '../../src/mediaEvent'; +import { createConnectedEventWithOneEndpoint, mockTrack } from '../fixtures'; +import { mockMediaStream, mockRTCPeerConnection } from '../mocks'; + +it('getLocalTrackAudioLevel returns null for unknown track id', async () => { + const webRTCEndpoint = new WebRTCEndpoint(); + const serializedEvent = serializeServerMediaEvent({ connected: createConnectedEventWithOneEndpoint() }); + webRTCEndpoint.receiveMediaEvent(serializedEvent); + + const result = await webRTCEndpoint.getLocalTrackAudioLevel('non-existent-track-id'); + + expect(result).toBeNull(); +}); + +it('getLocalTrackAudioLevel returns null when track has no sender', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + + const webRTCEndpoint = new WebRTCEndpoint(); + const serializedEvent = serializeServerMediaEvent({ connected: createConnectedEventWithOneEndpoint() }); + webRTCEndpoint.receiveMediaEvent(serializedEvent); + webRTCEndpoint.addTrack(mockTrack); + + const [trackId] = Object.keys(webRTCEndpoint['local']['localTracks']); + + const result = await webRTCEndpoint.getLocalTrackAudioLevel(trackId!); + + // sender is not set until offer/answer exchange, so getAudioLevel returns null + expect(result).toBeNull(); +}); + +it('getLocalTrackAudioLevel returns audio level from sender stats', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + + const webRTCEndpoint = new WebRTCEndpoint(); + const serializedEvent = serializeServerMediaEvent({ connected: createConnectedEventWithOneEndpoint() }); + webRTCEndpoint.receiveMediaEvent(serializedEvent); + webRTCEndpoint.addTrack(mockTrack); + + const [trackId] = Object.keys(webRTCEndpoint['local']['localTracks']); + const localTrack = webRTCEndpoint['local']['localTracks'][trackId!]; + + const statsMap = new Map([['report-1', { type: 'media-source', kind: 'audio', audioLevel: 0.42 }]]); + localTrack!['sender'] = { getStats: vi.fn().mockResolvedValue(statsMap) } as any; + + const result = await webRTCEndpoint.getLocalTrackAudioLevel(trackId!); + + expect(result).toEqual({ level: 0.42 }); +}); + +it('getLocalTrackAudioLevel returns null when stats have no audio media-source report', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + + const webRTCEndpoint = new WebRTCEndpoint(); + const serializedEvent = serializeServerMediaEvent({ connected: createConnectedEventWithOneEndpoint() }); + webRTCEndpoint.receiveMediaEvent(serializedEvent); + webRTCEndpoint.addTrack(mockTrack); + + const [trackId] = Object.keys(webRTCEndpoint['local']['localTracks']); + const localTrack = webRTCEndpoint['local']['localTracks'][trackId!]; + + const statsMap = new Map([['report-1', { type: 'media-source', kind: 'video', videoWidth: 1280 }]]); + localTrack!['sender'] = { getStats: vi.fn().mockResolvedValue(statsMap) } as any; + + const result = await webRTCEndpoint.getLocalTrackAudioLevel(trackId!); + + expect(result).toBeNull(); +}); From 5c309ea8e5054c72c889d537a81b25a0eefc60d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 17:09:18 +0100 Subject: [PATCH 10/18] Add more docs --- .../fishjam-chat/components/VideosGrid.tsx | 1 - packages/ts-client/src/FishjamClient.ts | 14 ++++++++++++++ packages/webrtc-client/src/webRTCEndpoint.ts | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx index 6053feb96..09e48e8ce 100644 --- a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx +++ b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx @@ -1,4 +1,3 @@ -import type { PeerId } from '@fishjam-cloud/react-native-client'; import { RTCView, usePeers, useVAD } from '@fishjam-cloud/react-native-client'; import React, { useCallback, useMemo } from 'react'; import type { ListRenderItemInfo } from 'react-native'; diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index 09d8b96e9..b15c84e50 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -949,6 +949,20 @@ export class FishjamClient { return this.webrtc?.getLocalTrackAudioLevel(trackId) ?? Promise.resolve(null); } diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index 25477a172..701f8e9d1 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -119,6 +119,21 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter { return this.local.getLocalTrackAudioLevel(trackId); } From b84a6d0b04f0876b8a888674265e554d328f1b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 17:27:32 +0100 Subject: [PATCH 11/18] Fix link issue --- packages/ts-client/src/FishjamClient.ts | 4 ++-- packages/webrtc-client/src/webRTCEndpoint.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index b15c84e50..88a3f87b9 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -952,7 +952,7 @@ export class FishjamClient { diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index 701f8e9d1..a1d49a2ca 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -132,7 +132,7 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter { return this.local.getLocalTrackAudioLevel(trackId); From afb9791ab1bea34d27632e47a6378a7e6c6d397d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 17:51:15 +0100 Subject: [PATCH 12/18] Prevent bubbling up of the error --- .../webrtc-client/src/tracks/LocalTrack.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/webrtc-client/src/tracks/LocalTrack.ts b/packages/webrtc-client/src/tracks/LocalTrack.ts index f5f90d448..f6e191b32 100644 --- a/packages/webrtc-client/src/tracks/LocalTrack.ts +++ b/packages/webrtc-client/src/tracks/LocalTrack.ts @@ -249,17 +249,17 @@ export class LocalTrack implements TrackCommon { }; public getAudioLevel = async (): Promise<{ level: number } | null> => { - if (!this.sender) { - return null; - } + if (!this.sender) return null; - const stats = await this.sender.getStats(); - for (const report of stats.values()) { - if (report.type === 'media-source' && report.kind === 'audio' && typeof report.audioLevel === 'number') { - return { level: report.audioLevel }; - } + try { + const stats = await this.sender.getStats(); + const source = [...stats.values()].find( + (r) => r.type === 'media-source' && r.kind === 'audio' && typeof r.audioLevel === 'number', + ); + return source ? { level: source.audioLevel } : null; + } catch { + return null; } - return null; }; public createTrackVariantBitratesEvent = () => { From aa8e8ce7fba7a9d579b23cdcf48be80c7559a3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Tue, 24 Mar 2026 18:00:58 +0100 Subject: [PATCH 13/18] Fix trackAudio being null potentially making localVAD permantently active --- packages/react-client/src/hooks/useLocalVAD.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index e09e47d52..db9eba37e 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -40,8 +40,8 @@ export const useLocalVAD = (showLocalPeer: boolean): Record => polling = true; try { const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrackId); - if (cancelled || trackAudio == null) return; - if (trackAudio.level > THRESHOLD) { + if (cancelled) return; + if (trackAudio != null && trackAudio.level > THRESHOLD) { silenceTicks = 0; if (!isSpeech) { isSpeech = true; From 10f75b325139dc7b9a9c41962d80d66bb866835f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Wed, 25 Mar 2026 10:05:53 +0100 Subject: [PATCH 14/18] Remove redundant comment --- packages/react-client/src/hooks/useLocalVAD.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index db9eba37e..1c7af0eff 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -28,7 +28,6 @@ export const useLocalVAD = (showLocalPeer: boolean): Record => useEffect(() => { if (!showLocalPeer || !localPeerId || !microphoneTrackId) return; - // above -32 dBov -> speech, below -> silence; convert dBov to linear [0, 1] scale: const THRESHOLD = 10 ** (-32 / 20); const SILENCE_DEBOUNCE_TICKS = 2; let silenceTicks = 0; From 4401db5dfe842c4bc746bc7a13055768df3dc4dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Wed, 25 Mar 2026 13:43:49 +0100 Subject: [PATCH 15/18] Fix issues pointed out in PR --- .../fishjam-chat/components/VideosGrid.tsx | 11 +-- .../react-client/src/hooks/useLocalVAD.ts | 67 +++++++++---------- packages/react-client/src/hooks/useVAD.ts | 4 +- 3 files changed, 39 insertions(+), 43 deletions(-) diff --git a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx index 09e48e8ce..d5fc4020b 100644 --- a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx +++ b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx @@ -11,11 +11,9 @@ import NoCameraView from './NoCameraView'; const GridTrackItem = ({ peer, _index, - isSpeaking, }: { peer: GridTrack; _index: number; - isSpeaking: boolean; }) => { const isSelfVideo = peer.isLocal && peer.track?.metadata?.type === 'camera'; const isCamera = peer.track?.metadata?.type === 'camera'; @@ -23,6 +21,9 @@ const GridTrackItem = ({ peer.track?.stream && !peer.track?.metadata?.paused ? peer.track.stream : null; + const vadStatus = useVAD({ peerIds: [peer.peerId] }); + const isPeerSpeaking = + vadStatus[peer.peerId] && peer.track?.metadata?.type === 'camera'; return ( @@ -33,8 +34,10 @@ const GridTrackItem = ({ backgroundColor: peer.isLocal ? BrandColors.seaBlue60 : BrandColors.darkBlue60, - borderColor: isSpeaking ? '#00ff1a' : BrandColors.darkBlue100, - borderWidth: 2, + borderColor: isPeerSpeaking + ? BrandColors.seaBlue80 + : BrandColors.darkBlue100, + borderWidth: isPeerSpeaking ? 3 : 2, }, ]}> {mediaStream ? ( diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index 1c7af0eff..2ad929ee9 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -4,6 +4,14 @@ import { FishjamClientContext } from "../contexts/fishjamClient"; import type { PeerId } from "../types/public"; import { usePeers } from "./usePeers"; +// This is a dBov-to-linear conversion. -32 dBov number is taken from backend VAD threshold +// formula for dBov to linear conversion: linear = 10 ^ (dBov / 20) +// So -32 dBov = 10^(-32/20) ≈ 0.025. This is the minimum audio level considered "speech". +const THRESHOLD = 10 ** (-32 / 20); + +// Number of consecutive "silence" ticks before we consider speech to have stopped. Helps with smoothing out brief pauses in speech. +const SILENCE_DEBOUNCE_TICKS = 2; + /** * Client-side voice activity detection for the local peer. * @@ -19,7 +27,7 @@ import { usePeers } from "./usePeers"; * @returns A record mapping the local peer's id to its current speaking state, * or an empty object if `showLocalPeer` is false or no microphone track is found. */ -export const useLocalVAD = (showLocalPeer: boolean): Record => { +export const useLocalVAD = (options: { disabled: boolean }): Record => { const fishjamClient = useContext(FishjamClientContext); const [isSpeaking, setIsSpeaking] = useState(false); const { localPeer } = usePeers(); @@ -27,49 +35,34 @@ export const useLocalVAD = (showLocalPeer: boolean): Record => const microphoneTrackId = localPeer?.microphoneTrack?.trackId; useEffect(() => { - if (!showLocalPeer || !localPeerId || !microphoneTrackId) return; - const THRESHOLD = 10 ** (-32 / 20); - const SILENCE_DEBOUNCE_TICKS = 2; + if (options.disabled || !localPeerId || !microphoneTrackId) return; + let silenceTicks = 0; - let isSpeech = false; - let cancelled = false; - let polling = false; - const interval = setInterval(async () => { - if (cancelled || polling) return; - polling = true; - try { - const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrackId); - if (cancelled) return; - if (trackAudio != null && trackAudio.level > THRESHOLD) { - silenceTicks = 0; - if (!isSpeech) { - isSpeech = true; - setIsSpeaking(true); - } - } else { - silenceTicks += 1; - if (silenceTicks >= SILENCE_DEBOUNCE_TICKS && isSpeech) { - isSpeech = false; - setIsSpeaking(false); - } + let timeoutId: ReturnType; + + const poll = async () => { + const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrackId); + if (trackAudio != null && trackAudio.level > THRESHOLD) { + silenceTicks = 0; + setIsSpeaking(true); + } else { + silenceTicks += 1; + if (silenceTicks >= SILENCE_DEBOUNCE_TICKS) { + setIsSpeaking(false); } - } catch { - console.error("Error polling local track audio level for VAD", { - peerId: localPeerId, - trackId: microphoneTrackId, - }); - } finally { - polling = false; } - }, 100); + + timeoutId = setTimeout(poll, 100); + }; + + timeoutId = setTimeout(poll, 100); return () => { - cancelled = true; - clearInterval(interval); + clearTimeout(timeoutId); setIsSpeaking(false); }; - }, [showLocalPeer, fishjamClient, localPeerId, microphoneTrackId]); + }, [options.disabled, fishjamClient, localPeerId, microphoneTrackId]); - if (!localPeerId || !showLocalPeer || !microphoneTrackId) return {}; + if (!localPeerId || options.disabled || !microphoneTrackId) return {}; return { [localPeerId]: isSpeaking }; }; diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 12a953315..976295c8a 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -36,7 +36,7 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record @@ -83,7 +83,7 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record unsubs.forEach((unsub) => unsub()); }, [micTracksWithSelectedPeerIds]); - const localVAD = useLocalVAD(showLocalPeer); + const localVAD = useLocalVAD({ disabled: !showLocalPeerVAD }); const vadStatuses = useMemo( () => From cc543c6ead99efde208cd450ceea9037e2a343cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Wed, 25 Mar 2026 14:38:00 +0100 Subject: [PATCH 16/18] Fix stupid issues --- .../fishjam-chat/components/VideosGrid.tsx | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx index d5fc4020b..83e9723ee 100644 --- a/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx +++ b/examples/mobile-client/fishjam-chat/components/VideosGrid.tsx @@ -72,13 +72,6 @@ type VideosGridProps = { export default function VideosGrid({ username }: VideosGridProps) { const { localPeer, remotePeers } = usePeers(); const videoTracks = parsePeersToTracks(localPeer, remotePeers); - const vadPeers = useMemo(() => { - const remoteIds = remotePeers.map((peer) => peer.id); - return localPeer?.id ? [localPeer.id, ...remoteIds] : remoteIds; - }, [remotePeers, localPeer]); - const vadStates = useVAD({ - peerIds: vadPeers, - }); const keyExtractor = useCallback( (item: GridTrack, index: number) => item.track?.trackId ?? index.toString(), @@ -87,16 +80,9 @@ export default function VideosGrid({ username }: VideosGridProps) { const renderItem = useCallback( ({ item, index }: ListRenderItemInfo) => ( - + ), - [vadStates], + [], ); const ListEmptyComponent = useMemo( From 8b13b89f9c96be6953b660e0a591f5d88912bf1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Wed, 25 Mar 2026 14:50:17 +0100 Subject: [PATCH 17/18] Fix the issues i forgot to commit --- packages/react-client/src/hooks/useLocalVAD.ts | 2 +- packages/react-client/src/hooks/useVAD.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index 2ad929ee9..7e5cfd631 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -55,7 +55,7 @@ export const useLocalVAD = (options: { disabled: boolean }): Record { clearTimeout(timeoutId); diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 976295c8a..128223693 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -36,7 +36,10 @@ export const useVAD = (options: { peerIds: ReadonlyArray }): Record (clientState.localPeer?.id ? peerIds.includes(clientState.localPeer?.id) : false), + [clientState.localPeer?.id, peerIds], + ); const micTracksWithSelectedPeerIds = useMemo( () => From 039ebb5e15ff6a686943b568c3622b765ae77432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gadomski?= Date: Wed, 25 Mar 2026 14:57:28 +0100 Subject: [PATCH 18/18] Fix JSDocs --- packages/react-client/src/hooks/useLocalVAD.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts index 7e5cfd631..6ca4a3f59 100644 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ b/packages/react-client/src/hooks/useLocalVAD.ts @@ -25,7 +25,7 @@ const SILENCE_DEBOUNCE_TICKS = 2; * * @internal Used by `useVAD` when the local peer's id is included in `peerIds`. * @returns A record mapping the local peer's id to its current speaking state, - * or an empty object if `showLocalPeer` is false or no microphone track is found. + * or an empty object if `options.disabled` is true, the local peer is not available, or no microphone track is found. */ export const useLocalVAD = (options: { disabled: boolean }): Record => { const fishjamClient = useContext(FishjamClientContext);