common: extend audioOnlyMode to support on | off | allowScreenshareVideo

.changeset/common-clubs-fly.md

@@ -0,0 +1,6 @@
+---
+"@whereby.com/media": minor
+"@whereby.com/core": minor
+---
+
+extend audioOnlyMode to include off | on | allowScreenshareVideo

package.json

@@ -7,6 +7,12 @@
     "packageManager": "pnpm@10.12.4",
     "scripts": {
         "build": "turbo build",
+        "build:media": "turbo build --filter=@whereby.com/media",
+        "build:core": "turbo build --filter=@whereby.com/core",
+        "build:browser-sdk": "turbo build --filter=@whereby.com/browser-sdk",
+        "build:assistant-sdk": "turbo build --filter=@whereby.com/assistant-sdk",
+        "build:react-native-sdk": "turbo build --filter=@whereby.com/react-native-sdk",
+        "build:camera-effects": "turbo build --filter=@whereby.com/camera-effects",
         "changeset": "changeset",
         "version": "changeset version && pnpm install --lockfile-only",
         "dev": "turbo dev --filter=@whereby.com/browser-sdk",

packages/core/src/client/RoomConnection/index.ts

@@ -318,6 +318,7 @@ export class RoomConnectionClient extends BaseClient<RoomConnectionState, RoomCo
             assistantKey: this.options.assistantKey || null,
             isNodeSdk: this.options.isNodeSdk || false,
             isAudioRecorder: this.options.isAudioRecorder ?? false,
+            audioOnlyMode: this.options.audioOnlyMode,
         };
 
         this.store.dispatch(doAppStart(roomConfig));

packages/core/src/redux/slices/app.ts

@@ -2,6 +2,7 @@ import { PayloadAction, createSlice } from "@reduxjs/toolkit";
 import { RootState } from "../store";
 import type { LocalMediaOptions } from "./localMedia";
 import { coreVersion } from "../../version";
+import { AudioOnlyMode } from "@whereby.com/media";
 
 /**
  * Reducer
@@ -19,9 +20,11 @@ export interface AppConfig {
     roomKey: string | null;
     roomUrl: string;
     userAgent?: string;
+    audioOnlyMode?: AudioOnlyMode;
 }
 
 export interface AppState {
+    audioOnlyMode: AudioOnlyMode;
     displayName: string | null;
     externalId: string | null;
     ignoreBreakoutGroups: boolean;
@@ -37,6 +40,7 @@ export interface AppState {
 }
 
 export const initialState: AppState = {
+    audioOnlyMode: "off",
     displayName: null,
     externalId: null,
     ignoreBreakoutGroups: false,
@@ -95,3 +99,4 @@ export const selectAppExternalId = (state: RootState) => state.app.externalId;
 export const selectAppIsNodeSdk = (state: RootState) => state.app.isNodeSdk;
 export const selectAppInitialConfig = (state: RootState) => state.app.initialConfig;
 export const selectAppIgnoreBreakoutGroups = (state: RootState) => state.app.ignoreBreakoutGroups;
+export const selectAppAudioOnlyMode = (state: RootState) => state.app.audioOnlyMode;

packages/core/src/redux/slices/room.ts

@@ -126,11 +126,30 @@ export const doEndMeeting = createAppAuthorizedThunk(
 
 export const selectRoomIsLocked = (state: RootState) => state.room.isLocked;
 
+export const selectRemoteScreenshares = createSelector(selectRemoteParticipants, (remoteParticipants) => {
+    const screenshares: Screenshare[] = [];
+
+    for (const participant of remoteParticipants) {
+        if (participant.presentationStream) {
+            screenshares.push({
+                id: participant.presentationStream.id || `pres-${participant.id}`,
+                participantId: participant.id,
+                hasAudioTrack: participant.presentationStream.getTracks().some((track) => track.kind === "audio"),
+                breakoutGroup: participant.breakoutGroup,
+                stream: participant.presentationStream,
+                isLocal: false,
+            });
+        }
+    }
+
+    return screenshares;
+});
+
 export const selectScreenshares = createSelector(
     selectLocalScreenshareStream,
     selectLocalParticipantRaw,
-    selectRemoteParticipants,
-    (localScreenshareStream, localParticipant, remoteParticipants) => {
+    selectRemoteScreenshares,
+    (localScreenshareStream, localParticipant, remoteScreenshares) => {
         const screenshares: Screenshare[] = [];
 
         if (localScreenshareStream) {
@@ -144,20 +163,7 @@ export const selectScreenshares = createSelector(
             });
         }
 
-        for (const participant of remoteParticipants) {
-            if (participant.presentationStream) {
-                screenshares.push({
-                    id: participant.presentationStream.id || `pres-${participant.id}`,
-                    participantId: participant.id,
-                    hasAudioTrack: participant.presentationStream.getTracks().some((track) => track.kind === "audio"),
-                    breakoutGroup: participant.breakoutGroup,
-                    stream: participant.presentationStream,
-                    isLocal: false,
-                });
-            }
-        }
-
-        return screenshares;
+        return screenshares.concat(remoteScreenshares);
     },
 );
 

packages/core/src/redux/slices/rtcConnection/index.ts

@@ -14,7 +14,13 @@ import { selectSignalConnectionRaw, selectSignalConnectionSocket, socketReconnec
 import { createReactor, startAppListening } from "../../listenerMiddleware";
 import { selectRemoteClients, selectRemoteParticipants, streamStatusUpdated } from "../remoteParticipants";
 import { RemoteParticipant, StreamState } from "../../../RoomParticipant";
-import { selectAppIsNodeSdk, selectAppIsActive, doAppStop, selectAppIgnoreBreakoutGroups } from "../app";
+import {
+    selectAppIsNodeSdk,
+    selectAppIsActive,
+    doAppStop,
+    selectAppIgnoreBreakoutGroups,
+    selectAppAudioOnlyMode,
+} from "../app";
 
 import {
     selectIsCameraEnabled,
@@ -44,6 +50,7 @@ function isDeferrable({ client, breakoutCurrentId }: { client?: RemoteParticipan
     return false;
 }
 import { rtcEvents } from "./actions";
+import { selectRemoteScreenshares } from "../room";
 export { rtcEvents } from "./actions";
 
 export const createWebRtcEmitter = (dispatch: AppDispatch) => {
@@ -208,6 +215,7 @@ export const doConnectRtc = createAppThunk(() => (dispatch, getState) => {
     const isCameraEnabled = selectIsCameraEnabled(state);
     const isMicrophoneEnabled = selectIsMicrophoneEnabled(state);
     const isNodeSdk = selectAppIsNodeSdk(state);
+    const audioOnlyMode = selectAppAudioOnlyMode(state);
 
     if (dispatcher || !socket) {
         return;
@@ -227,6 +235,7 @@ export const doConnectRtc = createAppThunk(() => (dispatch, getState) => {
     };
 
     const rtcManagerDispatcher = new RtcManagerDispatcher({
+        audioOnlyMode,
         emitter: createWebRtcEmitter(dispatch),
         serverSocket: socket,
         webrtcProvider,
@@ -278,6 +287,7 @@ export const doHandleAcceptStreams = createAppThunk((payload: StreamStatusUpdate
             rtcManager.acceptNewStream({
                 streamId: streamId === CAMERA_STREAM_ID ? clientId : streamId,
                 clientId,
+                audioOnlyMode: "off",
             });
         } else if (state === "new_accept" || state === "old_accept") {
             // do nothing - let this be marked as done_accept as the rtcManager
@@ -328,7 +338,10 @@ export const doRtcManagerInitialize = createAppThunk(() => (dispatch, getState)
     const isMicrophoneEnabled = selectIsMicrophoneEnabled(getState());
 
     if (localMediaStream && rtcManager) {
-        rtcManager.addCameraStream(localMediaStream, { audioPaused: !isMicrophoneEnabled, videoPaused: !isCameraEnabled });
+        rtcManager.addCameraStream(localMediaStream, {
+            audioPaused: !isMicrophoneEnabled,
+            videoPaused: !isCameraEnabled,
+        });
     }
 
     dispatch(rtcManagerInitialized());
@@ -562,3 +575,13 @@ createReactor(
         }
     },
 );
+
+createReactor([selectRemoteScreenshares], ({ getState }, remoteScreenshares) => {
+    const rtcManager = selectRtcManager(getState());
+    if (rtcManager) {
+        const screenshareVideoTrackIds = remoteScreenshares
+            .flatMap((screenshare) => screenshare.stream?.getVideoTracks() ?? [])
+            .map((track) => track.id);
+        rtcManager.setRemoteScreenshareVideoTrackIds(screenshareVideoTrackIds);
+    }
+});

packages/core/src/redux/tests/store/rtcConnection.spec.ts

@@ -12,8 +12,7 @@ import MockMediaStream from "../../../__mocks__/MediaStream";
 import { RtcManagerDispatcher } from "@whereby.com/media";
 import { initialLocalMediaState } from "../../slices/localMedia";
 import { diff } from "deep-object-diff";
-import { coreVersion } from "../../../version";
-import { doAppStop } from "../../slices/app";
+import { doAppStop, initialState } from "../../slices/app";
 
 jest.mock("@whereby.com/media");
 
@@ -77,17 +76,7 @@ describe("actions", () => {
                     withSignalConnection: true,
                     initialState: {
                         app: {
-                            displayName: null,
-                            externalId: null,
-                            ignoreBreakoutGroups: false,
-                            isActive: false,
-                            isAssistant: false,
-                            isAudioRecorder: false,
-                            isDialIn: false,
-                            isNodeSdk: true,
-                            roomName: null,
-                            roomUrl: null,
-                            userAgent: `core:${coreVersion}`,
+                            ...initialState,
                         },
                     },
                 });
@@ -164,7 +153,10 @@ describe("actions", () => {
         store.dispatch(doRtcManagerInitialize());
 
         expect(mockRtcManager.addCameraStream).toHaveBeenCalledTimes(1);
-        expect(mockRtcManager.addCameraStream).toHaveBeenCalledWith(store.getState().localMedia.stream, { audioPaused:true, videoPaused: true});
+        expect(mockRtcManager.addCameraStream).toHaveBeenCalledWith(store.getState().localMedia.stream, {
+            audioPaused: true,
+            videoPaused: true,
+        });
         expect(store.getState().rtcConnection.rtcManagerInitialized).toBe(true);
     });
 });

packages/media/src/utils/getMediasoupDevice.ts

@@ -1,8 +1,9 @@
 import { detectDeviceAsync, Device } from "mediasoup-client";
 import { BuiltinHandlerName } from "mediasoup-client/lib/types";
+import { RtcManagerFeatures } from "../webrtc";
 
 type SupportedDevice = BuiltinHandlerName | "NodeJS" | undefined;
-export const getMediasoupDeviceAsync = async (features: Record<string, boolean | undefined>): Promise<Device> => {
+export const getMediasoupDeviceAsync = async (features: RtcManagerFeatures): Promise<Device> => {
     if (features.isNodeSdk) {
         return new Device({ handlerName: "Safari12" });
     }

packages/media/src/utils/iceServers.ts

@@ -1,4 +1,4 @@
-export const maybeTurnOnly = (iceConfig: any, features: { useOnlyTURN: string }) => {
+export const maybeTurnOnly = (iceConfig: any, features: { useOnlyTURN?: string }) => {
     if (!features.useOnlyTURN) {
         return;
     }
@@ -25,7 +25,7 @@ export const maybeTurnOnly = (iceConfig: any, features: { useOnlyTURN: string })
 
 export const external_stun_servers = (
     iceConfig: any,
-    features: { addGoogleStunServers: string; addCloudflareStunServers: string },
+    features: { addGoogleStunServers?: string; addCloudflareStunServers?: string },
 ) => {
     if (features.addGoogleStunServers) {
         iceConfig.iceServers = [