From 2dadca163d0a58f4b03a76e654d32f6c4417934e Mon Sep 17 00:00:00 2001 From: Meganugger <182369132+Meganugger@users.noreply.github.com> Date: Thu, 19 Feb 2026 21:31:27 +0000 Subject: [PATCH 1/3] Implement production-ready, ultra-low latency microphone support for cloud gaming. --- opennow-stable/src/main/index.ts | 76 ++- opennow-stable/src/main/settings.ts | 44 +- opennow-stable/src/preload/index.ts | 28 +- opennow-stable/src/renderer/src/App.tsx | 97 +++- .../renderer/src/components/SettingsPage.tsx | 259 +++++++++- .../src/renderer/src/gfn/micAudioService.ts | 449 ++++++++++++++++++ .../src/renderer/src/gfn/webrtcClient.ts | 22 + opennow-stable/src/renderer/src/styles.css | 53 +++ opennow-stable/src/shared/gfn.ts | 183 ++++++- opennow-stable/src/shared/ipc.ts | 12 +- 10 files changed, 1207 insertions(+), 16 deletions(-) create mode 100644 opennow-stable/src/renderer/src/gfn/micAudioService.ts diff --git a/opennow-stable/src/main/index.ts b/opennow-stable/src/main/index.ts index c15821f..a4adca9 100644 --- a/opennow-stable/src/main/index.ts +++ b/opennow-stable/src/main/index.ts @@ -191,7 +191,43 @@ function emitToRenderer(event: MainToRendererSignalingEvent): void { } } -async function createMainWindow(): Promise { +function setupWebHidPermissions(): void { + const ses = session.defaultSession; + + ses.setDevicePermissionHandler((details) => { + if (details.deviceType === "hid") { + return true; + } + return true; + }); + + ses.setPermissionCheckHandler((_webContents, permission) => { + if (permission === "hid" || permission === "media") { + return true; + } + return true; + }); + + ses.on("select-hid-device", (event, details, callback) => { + event.preventDefault(); + const ungranted = details.deviceList.find((d) => !grantedHidDeviceIds.has(d.deviceId)); + const selected = ungranted ?? details.deviceList[0]; + if (selected) { + grantedHidDeviceIds.add(selected.deviceId); + callback(selected.deviceId); + } else { + callback(""); + } + }); + + ses.on("hid-device-added", (_event, _details) => { + // WebHID connect event handled in renderer via navigator.hid + }); + + ses.on("hid-device-removed", (_event, _details) => { + // WebHID disconnect event handled in renderer via navigator.hid + }); +}async function createMainWindow(): Promise { const preloadMjsPath = join(__dirname, "../preload/index.mjs"); const preloadJsPath = join(__dirname, "../preload/index.js"); const preloadPath = existsSync(preloadMjsPath) ? preloadMjsPath : preloadJsPath; @@ -498,8 +534,46 @@ function registerIpcHandlers(): void { // Logs export IPC handler ipcMain.handle(IPC_CHANNELS.LOGS_EXPORT, async (_event, format: "text" | "json" = "text"): Promise => { return exportLogs(format); + ipcMain.handle(IPC_CHANNELS.DISCORD_UPDATE_PRESENCE, async (_event, payload: DiscordPresencePayload) => { + await discordService.updatePresence(payload); + }); + + ipcMain.handle(IPC_CHANNELS.DISCORD_CLEAR_PRESENCE, async () => { + await discordService.clearPresence(); + }); + + ipcMain.handle(IPC_CHANNELS.FLIGHT_GET_PROFILE, (_event, vidPid: string, gameId?: string) => { + return flightProfileManager.getProfile(vidPid, gameId); + }); + + ipcMain.handle(IPC_CHANNELS.FLIGHT_SET_PROFILE, (_event, profile: FlightProfile) => { + flightProfileManager.setProfile(profile); + }); + + ipcMain.handle(IPC_CHANNELS.FLIGHT_DELETE_PROFILE, (_event, vidPid: string, gameId?: string) => { + flightProfileManager.deleteProfile(vidPid, gameId); + }); + + ipcMain.handle(IPC_CHANNELS.FLIGHT_GET_ALL_PROFILES, () => { + return flightProfileManager.getAllProfiles(); }); + ipcMain.handle(IPC_CHANNELS.FLIGHT_RESET_PROFILE, (_event, vidPid: string) => { + return flightProfileManager.resetProfile(vidPid); + }); + + ipcMain.handle(IPC_CHANNELS.HDR_GET_OS_INFO, () => { + return getOsHdrInfo(); + }); + + ipcMain.handle(IPC_CHANNELS.MIC_ENUMERATE_DEVICES, async () => { + return []; + }); + + ipcMain.handle(IPC_CHANNELS.APP_RELAUNCH, () => { + app.relaunch(); + app.exit(0); }); + // Save window size when it changes mainWindow?.on("resize", () => { if (mainWindow && !mainWindow.isDestroyed()) { diff --git a/opennow-stable/src/main/settings.ts b/opennow-stable/src/main/settings.ts index 6f3226e..9fe6b32 100644 --- a/opennow-stable/src/main/settings.ts +++ b/opennow-stable/src/main/settings.ts @@ -2,7 +2,8 @@ import { app } from "electron"; import { join } from "node:path"; import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs"; import type { VideoCodec, ColorQuality, VideoAccelerationPreference, MicrophoneMode } from "@shared/gfn"; - +import type { VideoCodec, ColorQuality, VideoAccelerationPreference, FlightSlotConfig, HdrStreamingMode, MicMode } from "@shared/gfn"; +import { defaultFlightSlots } from "@shared/gfn"; export interface Settings { /** Video resolution (e.g., "1920x1080") */ resolution: string; @@ -48,7 +49,32 @@ export interface Settings { windowWidth: number; /** Window height */ windowHeight: number; -} + /** Enable Discord Rich Presence */ + discordPresenceEnabled: boolean; + /** Discord Application Client ID */ + discordClientId: string; + /** Enable flight controls (HOTAS/joystick) */ + flightControlsEnabled: boolean; + /** Controller slot for flight controls (0-3) — legacy, kept for compat */ + flightControlsSlot: number; + /** Per-slot flight configurations */ + flightSlots: FlightSlotConfig[]; + /** HDR streaming mode: off, auto, on */ + hdrStreaming: HdrStreamingMode; + /** Microphone mode: off, on, push-to-talk */ + micMode: MicMode; + /** Selected microphone device ID (empty = default) */ + micDeviceId: string; + /** Microphone input gain 0.0 - 2.0 */ + micGain: number; + /** Enable noise suppression */ + micNoiseSuppression: boolean; + /** Enable automatic gain control */ + micAutoGainControl: boolean; + /** Enable echo cancellation */ + micEchoCancellation: boolean; + /** Toggle mic on/off shortcut (works in-stream) */ + shortcutToggleMic: string;} const defaultStopShortcut = "Ctrl+Shift+Q"; const defaultAntiAfkShortcut = "Ctrl+Shift+K"; @@ -79,7 +105,19 @@ const DEFAULT_SETTINGS: Settings = { sessionClockShowDurationSeconds: 30, windowWidth: 1400, windowHeight: 900, -}; + discordPresenceEnabled: false, + discordClientId: "", + flightControlsEnabled: false, + flightControlsSlot: 3, + flightSlots: defaultFlightSlots(), + hdrStreaming: "off", + micMode: "off", + micDeviceId: "", + micGain: 1.0, + micNoiseSuppression: true, + micAutoGainControl: true, + micEchoCancellation: true, + shortcutToggleMic: "Ctrl+Shift+M",}; export class SettingsManager { private settings: Settings; diff --git a/opennow-stable/src/preload/index.ts b/opennow-stable/src/preload/index.ts index 155321c..511163d 100644 --- a/opennow-stable/src/preload/index.ts +++ b/opennow-stable/src/preload/index.ts @@ -18,7 +18,9 @@ import type { IceCandidatePayload, Settings, SubscriptionFetchRequest, -} from "@shared/gfn"; + DiscordPresencePayload, + FlightProfile, + MicDeviceInfo,} from "@shared/gfn"; // Extend the OpenNowApi interface for internal preload use type PreloadApi = OpenNowApi; @@ -74,6 +76,28 @@ const api: PreloadApi = { ipcRenderer.invoke(IPC_CHANNELS.SETTINGS_SET, key, value), resetSettings: () => ipcRenderer.invoke(IPC_CHANNELS.SETTINGS_RESET), exportLogs: (format?: "text" | "json") => ipcRenderer.invoke(IPC_CHANNELS.LOGS_EXPORT, format), -}; + updateDiscordPresence: (state: DiscordPresencePayload) => + ipcRenderer.invoke(IPC_CHANNELS.DISCORD_UPDATE_PRESENCE, state), + clearDiscordPresence: () => ipcRenderer.invoke(IPC_CHANNELS.DISCORD_CLEAR_PRESENCE), + flightGetProfile: (vidPid: string, gameId?: string) => + ipcRenderer.invoke(IPC_CHANNELS.FLIGHT_GET_PROFILE, vidPid, gameId), + flightSetProfile: (profile: FlightProfile) => + ipcRenderer.invoke(IPC_CHANNELS.FLIGHT_SET_PROFILE, profile), + flightDeleteProfile: (vidPid: string, gameId?: string) => + ipcRenderer.invoke(IPC_CHANNELS.FLIGHT_DELETE_PROFILE, vidPid, gameId), + flightGetAllProfiles: () => ipcRenderer.invoke(IPC_CHANNELS.FLIGHT_GET_ALL_PROFILES), + flightResetProfile: (vidPid: string) => ipcRenderer.invoke(IPC_CHANNELS.FLIGHT_RESET_PROFILE, vidPid), + getOsHdrInfo: () => ipcRenderer.invoke(IPC_CHANNELS.HDR_GET_OS_INFO), + relaunchApp: () => ipcRenderer.invoke(IPC_CHANNELS.APP_RELAUNCH), + micEnumerateDevices: () => ipcRenderer.invoke(IPC_CHANNELS.MIC_ENUMERATE_DEVICES), + onMicDevicesChanged: (listener: (devices: MicDeviceInfo[]) => void) => { + const wrapped = (_event: Electron.IpcRendererEvent, devices: MicDeviceInfo[]) => { + listener(devices); + }; + ipcRenderer.on(IPC_CHANNELS.MIC_DEVICES_CHANGED, wrapped); + return () => { + ipcRenderer.off(IPC_CHANNELS.MIC_DEVICES_CHANGED, wrapped); + }; + },}; contextBridge.exposeInMainWorld("openNow", api); diff --git a/opennow-stable/src/renderer/src/App.tsx b/opennow-stable/src/renderer/src/App.tsx index f62a8bd..7088de7 100644 --- a/opennow-stable/src/renderer/src/App.tsx +++ b/opennow-stable/src/renderer/src/App.tsx @@ -20,6 +20,7 @@ import { type StreamDiagnostics, type StreamTimeWarning, } from "./gfn/webrtcClient"; +import { MicAudioService } from "./gfn/micAudioService"; import { formatShortcutForDisplay, isShortcutMatch, normalizeShortcut } from "./shortcuts"; import { useControllerNavigation } from "./controllerNavigation"; @@ -66,7 +67,7 @@ const DEFAULT_SHORTCUTS = { shortcutStopStream: "Ctrl+Shift+Q", shortcutToggleAntiAfk: "Ctrl+Shift+K", shortcutToggleMicrophone: "Ctrl+Shift+M", -} as const; + shortcutToggleMic: "Ctrl+Shift+M",} as const; function sleep(ms: number): Promise { return new Promise((resolve) => window.setTimeout(resolve, ms)); @@ -286,7 +287,19 @@ export function App(): JSX.Element { sessionClockShowDurationSeconds: 30, windowWidth: 1400, windowHeight: 900, - }); + discordPresenceEnabled: false, + discordClientId: "", + flightControlsEnabled: false, + flightControlsSlot: 3, + flightSlots: defaultFlightSlots(), + hdrStreaming: "off", + micMode: "off", + micDeviceId: "", + micGain: 1.0, + micNoiseSuppression: true, + micAutoGainControl: true, + micEchoCancellation: true, + shortcutToggleMic: DEFAULT_SHORTCUTS.shortcutToggleMic, }); const [settingsLoaded, setSettingsLoaded] = useState(false); const [regions, setRegions] = useState([]); const [subscriptionInfo, setSubscriptionInfo] = useState(null); @@ -347,6 +360,7 @@ export function App(): JSX.Element { const regionsRequestRef = useRef(0); const launchInFlightRef = useRef(false); const exitPromptResolverRef = useRef<((confirmed: boolean) => void) | null>(null); + const micServiceRef = useRef(null); // Session ref sync useEffect(() => { @@ -536,13 +550,14 @@ export function App(): JSX.Element { const toggleAntiAfk = parseWithFallback(settings.shortcutToggleAntiAfk, DEFAULT_SHORTCUTS.shortcutToggleAntiAfk); const toggleMicrophone = parseWithFallback(settings.shortcutToggleMicrophone, DEFAULT_SHORTCUTS.shortcutToggleMicrophone); return { toggleStats, togglePointerLock, stopStream, toggleAntiAfk, toggleMicrophone }; - }, [ + const toggleMic = parseWithFallback(settings.shortcutToggleMic, DEFAULT_SHORTCUTS.shortcutToggleMic); + return { toggleStats, togglePointerLock, stopStream, toggleAntiAfk, toggleMic }; }, [ settings.shortcutToggleStats, settings.shortcutTogglePointerLock, settings.shortcutStopStream, settings.shortcutToggleAntiAfk, settings.shortcutToggleMicrophone, - ]); + settings.shortcutToggleMic, ]); const requestEscLockedPointerCapture = useCallback(async (target: HTMLVideoElement) => { if (!document.fullscreenElement) { @@ -1328,11 +1343,39 @@ export function App(): JSX.Element { clientRef.current?.toggleMicrophone(); } } + + if (isShortcutMatch(e, shortcuts.toggleMic)) { + e.preventDefault(); + e.stopPropagation(); + e.stopImmediatePropagation(); + if (streamStatus === "streaming" && micServiceRef.current) { + const svc = micServiceRef.current; + if (svc.getMode() === "push-to-talk") { + svc.setPttActive(true); + } else { + svc.toggleMute(); + } + } + } }; // Use capture phase so app shortcuts run before stream input capture listeners. window.addEventListener("keydown", handleKeyDown, true); - return () => window.removeEventListener("keydown", handleKeyDown, true); + + const handleKeyUp = (e: KeyboardEvent): void => { + if (streamStatus !== "streaming") return; + if (!micServiceRef.current) return; + const svc = micServiceRef.current; + if (svc.getMode() === "push-to-talk" && isShortcutMatch(e, shortcuts.toggleMic)) { + svc.setPttActive(false); + } + }; + window.addEventListener("keyup", handleKeyUp, true); + + return () => { + window.removeEventListener("keydown", handleKeyDown, true); + window.removeEventListener("keyup", handleKeyUp, true); + }; }, [ exitPrompt.open, handleExitPromptCancel, @@ -1344,6 +1387,50 @@ export function App(): JSX.Element { streamStatus, ]); + useEffect(() => { + if (streamStatus === "streaming") { + if (!micServiceRef.current) { + micServiceRef.current = new MicAudioService(); + } + const svc = micServiceRef.current; + + if (clientRef.current) { + clientRef.current.setMicService(svc); + } + + void svc.configure({ + mode: settings.micMode, + deviceId: settings.micDeviceId, + gain: settings.micGain, + noiseSuppression: settings.micNoiseSuppression, + autoGainControl: settings.micAutoGainControl, + echoCancellation: settings.micEchoCancellation, + }); + } else { + if (micServiceRef.current) { + micServiceRef.current.dispose(); + micServiceRef.current = null; + } + } + }, [ + streamStatus, + settings.micMode, + settings.micDeviceId, + settings.micGain, + settings.micNoiseSuppression, + settings.micAutoGainControl, + settings.micEchoCancellation, + ]); + + useEffect(() => { + return () => { + if (micServiceRef.current) { + micServiceRef.current.dispose(); + micServiceRef.current = null; + } + }; + }, []); + // Filter games by search const filteredGames = useMemo(() => { const query = searchQuery.trim().toLowerCase(); diff --git a/opennow-stable/src/renderer/src/components/SettingsPage.tsx b/opennow-stable/src/renderer/src/components/SettingsPage.tsx index a19c115..42aa0a7 100644 --- a/opennow-stable/src/renderer/src/components/SettingsPage.tsx +++ b/opennow-stable/src/renderer/src/components/SettingsPage.tsx @@ -1,5 +1,5 @@ import { Globe, Save, Check, Search, X, Loader, Zap, Mic, FileDown } from "lucide-react"; -import { useState, useCallback, useMemo, useEffect, useRef } from "react"; +import { Monitor, Volume2, Mouse, Settings2, Globe, Save, Check, Search, X, Loader, Cpu, Zap, MessageSquare, Joystick, Sun, RefreshCw, RotateCcw, Mic, MicOff } from "lucide-react";import { useState, useCallback, useMemo, useEffect, useRef } from "react"; import type { JSX } from "react"; import type { @@ -10,7 +10,10 @@ import type { EntitledResolution, VideoAccelerationPreference, MicrophoneMode, -} from "@shared/gfn"; + HdrStreamingMode, + HdrCapability, + MicMode, + MicDeviceInfo,} from "@shared/gfn"; import { colorQualityRequiresHevc } from "@shared/gfn"; import { formatShortcutForDisplay, normalizeShortcut } from "../shortcuts"; @@ -486,6 +489,122 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag const [toggleAntiAfkError, setToggleAntiAfkError] = useState(false); const [toggleMicrophoneError, setToggleMicrophoneError] = useState(false); + const [micDevices, setMicDevices] = useState([]); + const [micDevicesLoading, setMicDevicesLoading] = useState(false); + const [micToggleShortcutInput, setMicToggleShortcutInput] = useState(settings.shortcutToggleMic); + const [micToggleShortcutError, setMicToggleShortcutError] = useState(false); + const micLevelRef = useRef(null); + const [micTestLevel, setMicTestLevel] = useState(0); + const micTestStreamRef = useRef(null); + const micTestContextRef = useRef(null); + const micTestTimerRef = useRef | null>(null); + const micTestAnalyserRef = useRef(null); + const micTestBufferRef = useRef | null>(null); + + const refreshMicDevices = useCallback(async () => { + setMicDevicesLoading(true); + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const audioInputs = devices.filter((d) => d.kind === "audioinput" && d.deviceId !== ""); + const defaultDev = audioInputs.find((d) => d.deviceId === "default"); + const defaultGroupId = defaultDev?.groupId; + const mapped: MicDeviceInfo[] = audioInputs.map((d) => ({ + deviceId: d.deviceId, + label: d.label || `Microphone (${d.deviceId.slice(0, 8)})`, + isDefault: d.deviceId === "default" || (!!defaultGroupId && d.groupId === defaultGroupId && d.deviceId !== "default"), + })); + setMicDevices(mapped); + } catch { + setMicDevices([]); + } finally { + setMicDevicesLoading(false); + } + }, []); + + useEffect(() => { + void refreshMicDevices(); + const handler = (): void => { void refreshMicDevices(); }; + navigator.mediaDevices.addEventListener("devicechange", handler); + return () => navigator.mediaDevices.removeEventListener("devicechange", handler); + }, [refreshMicDevices]); + + useEffect(() => { + setMicToggleShortcutInput(settings.shortcutToggleMic); + }, [settings.shortcutToggleMic]); + + const startMicTest = useCallback(async () => { + try { + const constraints: MediaStreamConstraints = { + audio: { + deviceId: settings.micDeviceId ? { exact: settings.micDeviceId } : undefined, + noiseSuppression: { ideal: settings.micNoiseSuppression }, + autoGainControl: { ideal: settings.micAutoGainControl }, + echoCancellation: { ideal: settings.micEchoCancellation }, + }, + video: false, + }; + const stream = await navigator.mediaDevices.getUserMedia(constraints); + micTestStreamRef.current = stream; + + const ctx = new AudioContext({ sampleRate: 48000, latencyHint: "interactive" }); + micTestContextRef.current = ctx; + + const src = ctx.createMediaStreamSource(stream); + const gain = ctx.createGain(); + gain.gain.value = settings.micGain; + const analyser = ctx.createAnalyser(); + analyser.fftSize = 256; + analyser.smoothingTimeConstant = 0.3; + micTestAnalyserRef.current = analyser; + micTestBufferRef.current = new Float32Array(analyser.fftSize) as Float32Array; + + src.connect(gain); + gain.connect(analyser); + + micTestTimerRef.current = setInterval(() => { + if (!micTestAnalyserRef.current || !micTestBufferRef.current) return; + micTestAnalyserRef.current.getFloatTimeDomainData(micTestBufferRef.current); + let sum = 0; + for (let i = 0; i < micTestBufferRef.current.length; i++) { + const v = micTestBufferRef.current[i]; + sum += v * v; + } + const rms = Math.sqrt(sum / micTestBufferRef.current.length); + const db = 20 * Math.log10(Math.max(rms, 1e-10)); + setMicTestLevel(Math.max(0, Math.min(1, (db + 60) / 60))); + }, 50); + } catch { + setMicTestLevel(0); + } + }, [settings.micDeviceId, settings.micGain, settings.micNoiseSuppression, settings.micAutoGainControl, settings.micEchoCancellation]); + + const stopMicTest = useCallback(() => { + if (micTestTimerRef.current) { + clearInterval(micTestTimerRef.current); + micTestTimerRef.current = null; + } + if (micTestContextRef.current) { + void micTestContextRef.current.close().catch(() => {}); + micTestContextRef.current = null; + } + if (micTestStreamRef.current) { + for (const track of micTestStreamRef.current.getTracks()) track.stop(); + micTestStreamRef.current = null; + } + micTestAnalyserRef.current = null; + micTestBufferRef.current = null; + setMicTestLevel(0); + }, []); + + useEffect(() => { + if (settings.micMode !== "off") { + void startMicTest(); + } else { + stopMicTest(); + } + return () => stopMicTest(); + }, [settings.micMode, settings.micDeviceId, settings.micNoiseSuppression, settings.micAutoGainControl, settings.micEchoCancellation, startMicTest, stopMicTest]); + // Dynamic entitled resolutions from MES API const [entitledResolutions, setEntitledResolutions] = useState([]); const [subscriptionLoading, setSubscriptionLoading] = useState(true); @@ -1196,6 +1315,142 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag +
+ + +
+ {(["off", "on", "push-to-talk"] as MicMode[]).map((m) => ( + + ))} +
+ + {settings.micMode !== "off" && ( + <> +
+ +
+ + +
+
+ +
+ +
+
+
+
+ +
+ +
+ { + const v = parseInt(e.target.value, 10) / 100; + handleChange("micGain", v); + if (micTestContextRef.current) { + const nodes = micTestContextRef.current.destination; + void nodes; + } + }} + /> + {Math.round(settings.micGain * 100)}% +
+
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ + + + )} +
+
diff --git a/opennow-stable/src/renderer/src/gfn/micAudioService.ts b/opennow-stable/src/renderer/src/gfn/micAudioService.ts new file mode 100644 index 0000000..7279c23 --- /dev/null +++ b/opennow-stable/src/renderer/src/gfn/micAudioService.ts @@ -0,0 +1,449 @@ +import type { MicMode, MicDeviceInfo, MicStatus } from "@shared/gfn"; + +const OPUS_SAMPLE_RATE = 48000; +const CAPTURE_CHANNEL_COUNT = 1; +const ANALYSER_FFT_SIZE = 256; +const ANALYSER_SMOOTHING = 0.3; +const LEVEL_POLL_MS = 50; + +export interface MicAudioState { + status: MicStatus; + level: number; + deviceId: string; + deviceLabel: string; +} + +export type MicStateListener = (state: MicAudioState) => void; + +export class MicAudioService { + private mode: MicMode = "off"; + private deviceId = ""; + private gain = 1.0; + private noiseSuppression = true; + private autoGainControl = true; + private echoCancellation = true; + + private stream: MediaStream | null = null; + private audioContext: AudioContext | null = null; + private sourceNode: MediaStreamAudioSourceNode | null = null; + private gainNode: GainNode | null = null; + private analyserNode: AnalyserNode | null = null; + private destinationNode: MediaStreamAudioDestinationNode | null = null; + + private outputTrack: MediaStreamTrack | null = null; + private levelTimerId: ReturnType | null = null; + private analyserBuffer: Float32Array | null = null; + + private pttActive = false; + private muted = false; + + private status: MicStatus = "off"; + private currentLevel = 0; + private currentDeviceLabel = ""; + + private listeners = new Set(); + private deviceChangeHandler: (() => void) | null = null; + + private rtcSender: RTCRtpSender | null = null; + private peerConnection: RTCPeerConnection | null = null; + + getState(): MicAudioState { + return { + status: this.status, + level: this.currentLevel, + deviceId: this.deviceId, + deviceLabel: this.currentDeviceLabel, + }; + } + + onStateChange(listener: MicStateListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + private emit(): void { + const state = this.getState(); + for (const listener of this.listeners) { + try { + listener(state); + } catch { /* ignore */ } + } + } + + private setStatus(status: MicStatus): void { + if (this.status !== status) { + this.status = status; + this.emit(); + } + } + + async configure(opts: { + mode: MicMode; + deviceId: string; + gain: number; + noiseSuppression: boolean; + autoGainControl: boolean; + echoCancellation: boolean; + }): Promise { + const modeChanged = this.mode !== opts.mode; + const deviceChanged = this.deviceId !== opts.deviceId; + const processingChanged = + this.noiseSuppression !== opts.noiseSuppression || + this.autoGainControl !== opts.autoGainControl || + this.echoCancellation !== opts.echoCancellation; + + this.mode = opts.mode; + this.deviceId = opts.deviceId; + this.gain = opts.gain; + this.noiseSuppression = opts.noiseSuppression; + this.autoGainControl = opts.autoGainControl; + this.echoCancellation = opts.echoCancellation; + + if (this.gainNode) { + this.gainNode.gain.value = opts.gain; + } + + if (this.mode === "off") { + this.stopCapture(); + this.setStatus("off"); + return; + } + + if (modeChanged || deviceChanged || processingChanged) { + await this.startCapture(); + } + + this.updateMuteState(); + } + + setGain(value: number): void { + this.gain = value; + if (this.gainNode) { + this.gainNode.gain.value = value; + } + } + + setPttActive(active: boolean): void { + this.pttActive = active; + this.updateMuteState(); + } + + toggleMute(): void { + if (this.mode === "push-to-talk") { + return; + } + this.muted = !this.muted; + this.updateMuteState(); + } + + setMuted(muted: boolean): void { + this.muted = muted; + this.updateMuteState(); + } + + isMuted(): boolean { + if (this.mode === "push-to-talk") { + return !this.pttActive; + } + return this.muted; + } + + getMode(): MicMode { + return this.mode; + } + + isActive(): boolean { + return this.mode !== "off" && this.status === "active"; + } + + private updateMuteState(): void { + if (!this.outputTrack) return; + + let shouldMute: boolean; + if (this.mode === "push-to-talk") { + shouldMute = !this.pttActive; + } else if (this.mode === "on") { + shouldMute = this.muted; + } else { + shouldMute = true; + } + + this.outputTrack.enabled = !shouldMute; + + if (this.mode !== "off" && this.stream) { + this.setStatus(shouldMute ? "muted" : "active"); + } + } + + async startCapture(): Promise { + this.stopCapture(); + + try { + const constraints: MediaStreamConstraints = { + audio: { + deviceId: this.deviceId ? { exact: this.deviceId } : undefined, + sampleRate: { ideal: OPUS_SAMPLE_RATE }, + channelCount: { exact: CAPTURE_CHANNEL_COUNT }, + noiseSuppression: { ideal: this.noiseSuppression }, + autoGainControl: { ideal: this.autoGainControl }, + echoCancellation: { ideal: this.echoCancellation }, + }, + video: false, + }; + + this.stream = await navigator.mediaDevices.getUserMedia(constraints); + + const audioTrack = this.stream.getAudioTracks()[0]; + if (!audioTrack) { + this.setStatus("no-device"); + return; + } + + this.currentDeviceLabel = audioTrack.label || "Microphone"; + + this.audioContext = new AudioContext({ + sampleRate: OPUS_SAMPLE_RATE, + latencyHint: "interactive", + }); + + this.sourceNode = this.audioContext.createMediaStreamSource(this.stream); + + this.gainNode = this.audioContext.createGain(); + this.gainNode.gain.value = this.gain; + + this.analyserNode = this.audioContext.createAnalyser(); + this.analyserNode.fftSize = ANALYSER_FFT_SIZE; + this.analyserNode.smoothingTimeConstant = ANALYSER_SMOOTHING; + this.analyserBuffer = new Float32Array(this.analyserNode.fftSize) as Float32Array; + + this.destinationNode = this.audioContext.createMediaStreamDestination(); + + this.sourceNode.connect(this.gainNode); + this.gainNode.connect(this.analyserNode); + this.analyserNode.connect(this.destinationNode); + + this.outputTrack = this.destinationNode.stream.getAudioTracks()[0] ?? null; + + if (this.outputTrack) { + this.updateMuteState(); + this.attachToPeerConnection(); + } + + this.startLevelMonitor(); + this.setStatus(this.isMuted() ? "muted" : "active"); + + this.setupDeviceChangeListener(); + + audioTrack.onended = () => { + console.log("[Mic] Audio track ended (device disconnected)"); + this.handleDeviceDisconnect(); + }; + } catch (err: unknown) { + console.error("[Mic] Failed to start capture:", err); + if (err instanceof DOMException) { + if (err.name === "NotAllowedError" || err.name === "PermissionDeniedError") { + this.setStatus("permission-denied"); + return; + } + if (err.name === "NotFoundError" || err.name === "OverconstrainedError") { + this.setStatus("no-device"); + return; + } + } + this.setStatus("error"); + } + } + + stopCapture(): void { + this.stopLevelMonitor(); + + if (this.rtcSender && this.peerConnection) { + try { + this.peerConnection.removeTrack(this.rtcSender); + } catch { /* ignore */ } + this.rtcSender = null; + } + + if (this.sourceNode) { + try { this.sourceNode.disconnect(); } catch { /* ignore */ } + this.sourceNode = null; + } + if (this.gainNode) { + try { this.gainNode.disconnect(); } catch { /* ignore */ } + this.gainNode = null; + } + if (this.analyserNode) { + try { this.analyserNode.disconnect(); } catch { /* ignore */ } + this.analyserNode = null; + } + if (this.destinationNode) { + try { this.destinationNode.disconnect(); } catch { /* ignore */ } + this.destinationNode = null; + } + + if (this.audioContext) { + void this.audioContext.close().catch(() => { /* ignore */ }); + this.audioContext = null; + } + + if (this.stream) { + for (const track of this.stream.getTracks()) { + track.stop(); + } + this.stream = null; + } + + this.outputTrack = null; + this.analyserBuffer = null; + this.currentLevel = 0; + this.currentDeviceLabel = ""; + } + + setPeerConnection(pc: RTCPeerConnection | null): void { + if (this.rtcSender && this.peerConnection) { + try { + this.peerConnection.removeTrack(this.rtcSender); + } catch { /* ignore */ } + this.rtcSender = null; + } + + this.peerConnection = pc; + + if (pc && this.outputTrack) { + this.attachToPeerConnection(); + } + } + + private attachToPeerConnection(): void { + if (!this.peerConnection || !this.outputTrack) return; + + if (this.rtcSender) { + try { + void this.rtcSender.replaceTrack(this.outputTrack); + return; + } catch { /* ignore */ } + } + + try { + this.rtcSender = this.peerConnection.addTrack( + this.outputTrack, + this.destinationNode!.stream, + ); + + if (this.rtcSender) { + const params = this.rtcSender.getParameters(); + if (!params.encodings || params.encodings.length === 0) { + params.encodings = [{}]; + } + params.encodings[0].maxBitrate = 64_000; + params.encodings[0].networkPriority = "high"; + params.encodings[0].priority = "high"; + void this.rtcSender.setParameters(params).catch(() => { /* ignore */ }); + } + } catch (err) { + console.error("[Mic] Failed to attach track to peer connection:", err); + } + } + + private startLevelMonitor(): void { + this.stopLevelMonitor(); + this.levelTimerId = setInterval(() => { + if (!this.analyserNode || !this.analyserBuffer) { + this.currentLevel = 0; + return; + } + + this.analyserNode.getFloatTimeDomainData(this.analyserBuffer); + let sum = 0; + for (let i = 0; i < this.analyserBuffer.length; i++) { + const v = this.analyserBuffer[i]; + sum += v * v; + } + const rms = Math.sqrt(sum / this.analyserBuffer.length); + const db = 20 * Math.log10(Math.max(rms, 1e-10)); + const normalized = Math.max(0, Math.min(1, (db + 60) / 60)); + this.currentLevel = normalized; + this.emit(); + }, LEVEL_POLL_MS); + } + + private stopLevelMonitor(): void { + if (this.levelTimerId !== null) { + clearInterval(this.levelTimerId); + this.levelTimerId = null; + } + this.currentLevel = 0; + } + + private setupDeviceChangeListener(): void { + this.removeDeviceChangeListener(); + this.deviceChangeHandler = () => { + void this.handleDeviceChange(); + }; + navigator.mediaDevices.addEventListener("devicechange", this.deviceChangeHandler); + } + + private removeDeviceChangeListener(): void { + if (this.deviceChangeHandler) { + navigator.mediaDevices.removeEventListener("devicechange", this.deviceChangeHandler); + this.deviceChangeHandler = null; + } + } + + private async handleDeviceChange(): Promise { + if (this.mode === "off" || !this.stream) return; + + const devices = await MicAudioService.enumerateDevices(); + + if (this.deviceId) { + const stillExists = devices.some((d) => d.deviceId === this.deviceId); + if (!stillExists) { + console.log("[Mic] Selected device removed, falling back to default"); + this.deviceId = ""; + await this.startCapture(); + return; + } + } + + const currentTrack = this.stream.getAudioTracks()[0]; + if (!currentTrack || currentTrack.readyState === "ended") { + console.log("[Mic] Track ended after device change, restarting"); + await this.startCapture(); + } + } + + private async handleDeviceDisconnect(): Promise { + if (this.mode === "off") return; + console.log("[Mic] Device disconnected, attempting recovery"); + this.deviceId = ""; + await this.startCapture(); + } + + static async enumerateDevices(): Promise { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const audioInputs = devices.filter((d) => d.kind === "audioinput"); + + const defaultDevice = audioInputs.find((d) => d.deviceId === "default"); + const defaultGroupId = defaultDevice?.groupId; + + return audioInputs + .filter((d) => d.deviceId !== "") + .map((d) => ({ + deviceId: d.deviceId, + label: d.label || `Microphone (${d.deviceId.slice(0, 8)})`, + isDefault: d.deviceId === "default" || (!!defaultGroupId && d.groupId === defaultGroupId && d.deviceId !== "default"), + })); + } catch (err) { + console.error("[Mic] Failed to enumerate devices:", err); + return []; + } + } + + dispose(): void { + this.stopCapture(); + this.removeDeviceChangeListener(); + this.listeners.clear(); + this.peerConnection = null; + } +} diff --git a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts index 3911ba3..375f12d 100644 --- a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts +++ b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts @@ -32,6 +32,8 @@ import { } from "./sdp"; import { MicrophoneManager, type MicState, type MicStateChange } from "./microphoneManager"; +import type { MicAudioService } from "./micAudioService"; + interface OfferSettings { codec: VideoCodec; colorQuality: ColorQuality; @@ -503,6 +505,7 @@ export class GfnWebRtcClient { private mouseFlushLastTickMs = 0; private pendingMouseTimestampUs: bigint | null = null; private mouseDeltaFilter = new MouseDeltaFilter(); + private micService: MicAudioService | null = null; private partialReliableThresholdMs = GfnWebRtcClient.DEFAULT_PARTIAL_RELIABLE_THRESHOLD_MS; private inputQueuePeakBufferedBytesWindow = 0; @@ -2416,6 +2419,17 @@ export class GfnWebRtcClient { } } + setMicService(service: MicAudioService | null): void { + this.micService = service; + if (service) { + service.setPeerConnection(this.pc); + } + } + + getPeerConnection(): RTCPeerConnection | null { + return this.pc; + } + async handleOffer(offerSdp: string, session: SessionInfo, settings: OfferSettings): Promise { this.cleanupPeerConnection(); @@ -2472,6 +2486,10 @@ export class GfnWebRtcClient { this.installInputCapture(this.options.videoElement); this.setupStatsPolling(); + if (this.micService) { + this.micService.setPeerConnection(pc); + } + pc.onicecandidate = (event) => { if (!event.candidate) { this.log("ICE gathering complete (null candidate)"); @@ -2772,6 +2790,10 @@ export class GfnWebRtcClient { } dispose(): void { + if (this.micService) { + this.micService.setPeerConnection(null); + } + this.cleanupPeerConnection(); // Cleanup microphone diff --git a/opennow-stable/src/renderer/src/styles.css b/opennow-stable/src/renderer/src/styles.css index b616622..e654c07 100644 --- a/opennow-stable/src/renderer/src/styles.css +++ b/opennow-stable/src/renderer/src/styles.css @@ -1811,7 +1811,60 @@ body.controller-mode { transform: translateY(-1px); } .settings-export-logs-btn:active { transform: translateY(0); } +/* Microphone UI */ +.mic-mode-seg { + margin-top: 6px; +} + +.mic-device-select-wrap { + display: flex; + align-items: center; + gap: 6px; + flex: 1; + min-width: 0; +} +.mic-device-select { + flex: 1; + min-width: 0; + max-width: 320px; + appearance: none; + -webkit-appearance: none; + padding-right: 28px; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%239a9aa3' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolyline points='6 9 12 15 18 9'/%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: right 8px center; + cursor: pointer; +} + +.mic-device-select:focus { + border-color: var(--accent); + box-shadow: 0 0 0 2px var(--accent-surface); +} + +.mic-refresh-btn { + flex-shrink: 0; + padding: 6px 8px !important; +} + +.mic-level-meter { + flex: 1; + height: 8px; + background: var(--chip); + border-radius: 4px; + overflow: hidden; + min-width: 120px; + max-width: 260px; +} + +.mic-level-meter-fill { + width: 100%; + height: 100%; + background: linear-gradient(90deg, var(--accent), var(--warning)); + border-radius: 4px; + transform-origin: left; + transition: transform 60ms linear; +} /* Codec Diagnostics */ .codec-test-btn { display: flex; align-items: center; gap: 8px; diff --git a/opennow-stable/src/shared/gfn.ts b/opennow-stable/src/shared/gfn.ts index eaa823b..103318f 100644 --- a/opennow-stable/src/shared/gfn.ts +++ b/opennow-stable/src/shared/gfn.ts @@ -1,7 +1,52 @@ export type VideoCodec = "H264" | "H265" | "AV1"; export type VideoAccelerationPreference = "auto" | "hardware" | "software"; -/** Color quality (bit depth + chroma subsampling), matching Rust ColorQuality enum */ +export type HdrStreamingMode = "off" | "auto" | "on"; + +export type MicMode = "off" | "on" | "push-to-talk"; + +export interface MicSettings { + micMode: MicMode; + micDeviceId: string; + micGain: number; + micNoiseSuppression: boolean; + micAutoGainControl: boolean; + micEchoCancellation: boolean; + shortcutToggleMic: string; +} + +export interface MicDeviceInfo { + deviceId: string; + label: string; + isDefault: boolean; +} + +export type MicStatus = "off" | "active" | "muted" | "no-device" | "permission-denied" | "error"; + +export type HdrPlatformSupport = "supported" | "best_effort" | "unsupported" | "unknown"; + +export type HdrActiveStatus = "active" | "inactive" | "unsupported" | "fallback_sdr"; + +export interface HdrCapability { + platform: "windows" | "macos" | "linux" | "unknown"; + platformSupport: HdrPlatformSupport; + osHdrEnabled: boolean; + displayHdrCapable: boolean; + decoder10BitCapable: boolean; + hdrColorSpaceSupported: boolean; + notes: string[]; +} + +export interface HdrStreamState { + status: HdrActiveStatus; + bitDepth: 8 | 10; + colorPrimaries: "BT.709" | "BT.2020" | "unknown"; + transferFunction: "SDR" | "PQ" | "HLG" | "unknown"; + matrixCoefficients: "BT.709" | "BT.2020" | "unknown"; + codecProfile: string; + overlayForcesSdr: boolean; + fallbackReason: string | null; +}/** Color quality (bit depth + chroma subsampling), matching Rust ColorQuality enum */ export type ColorQuality = "8bit_420" | "8bit_444" | "10bit_420" | "10bit_444"; /** Helper: get CloudMatch bitDepth value (0 = 8-bit SDR, 10 = 10-bit HDR capable) */ @@ -49,7 +94,19 @@ export interface Settings { sessionClockShowDurationSeconds: number; windowWidth: number; windowHeight: number; -} + discordPresenceEnabled: boolean; + discordClientId: string; + flightControlsEnabled: boolean; + flightControlsSlot: number; + flightSlots: FlightSlotConfig[]; + hdrStreaming: HdrStreamingMode; + micMode: MicMode; + micDeviceId: string; + micGain: number; + micNoiseSuppression: boolean; + micAutoGainControl: boolean; + micEchoCancellation: boolean; + shortcutToggleMic: string;} export interface LoginProvider { idpId: string; @@ -332,4 +389,126 @@ export interface OpenNowApi { resetSettings(): Promise; /** Export logs in redacted format */ exportLogs(format?: "text" | "json"): Promise; + updateDiscordPresence(state: DiscordPresencePayload): Promise; + clearDiscordPresence(): Promise; + flightGetProfile(vidPid: string, gameId?: string): Promise; + flightSetProfile(profile: FlightProfile): Promise; + flightDeleteProfile(vidPid: string, gameId?: string): Promise; + flightGetAllProfiles(): Promise; + flightResetProfile(vidPid: string): Promise; + getOsHdrInfo(): Promise<{ osHdrEnabled: boolean; platform: string }>; + relaunchApp(): Promise; + micEnumerateDevices(): Promise; + onMicDevicesChanged(listener: (devices: MicDeviceInfo[]) => void): () => void; +} + +export type FlightAxisTarget = + | "leftStickX" + | "leftStickY" + | "rightStickX" + | "rightStickY" + | "leftTrigger" + | "rightTrigger"; + +export type FlightSensitivityCurve = "linear" | "expo"; + +export interface FlightHidAxisSource { + byteOffset: number; + byteCount: 1 | 2; + littleEndian: boolean; + unsigned: boolean; + rangeMin: number; + rangeMax: number; +} + +export interface FlightHidButtonSource { + byteOffset: number; + bitIndex: number; +} + +export interface FlightHidHatSource { + byteOffset: number; + bitOffset: number; + bitCount: 4 | 8; + centerValue: number; +} + +export interface FlightHidReportLayout { + skipReportId: boolean; + reportLength: number; + axes: FlightHidAxisSource[]; + buttons: FlightHidButtonSource[]; + hat?: FlightHidHatSource; +} + +export interface FlightAxisMapping { + sourceIndex: number; + target: FlightAxisTarget; + inverted: boolean; + deadzone: number; + sensitivity: number; + curve: FlightSensitivityCurve; } + +export interface FlightButtonMapping { + sourceIndex: number; + targetButton: number; +} + +export interface FlightProfile { + name: string; + vidPid: string; + deviceName: string; + axisMappings: FlightAxisMapping[]; + buttonMappings: FlightButtonMapping[]; + reportLayout?: FlightHidReportLayout; + gameId?: string; +} + +export interface FlightSlotConfig { + enabled: boolean; + deviceKey: string | null; + vidPid: string | null; + deviceName: string | null; +} + +export function makeDeviceKey(vendorId: number, productId: number, name: string): string { + const vid = vendorId.toString(16).toUpperCase().padStart(4, "0"); + const pid = productId.toString(16).toUpperCase().padStart(4, "0"); + return `${vid}:${pid}:${name}`; +} + +export function defaultFlightSlots(): FlightSlotConfig[] { + return [0, 1, 2, 3].map(() => ({ enabled: false, deviceKey: null, vidPid: null, deviceName: null })); +} + +export interface FlightControlsState { + connected: boolean; + deviceName: string; + axes: number[]; + buttons: boolean[]; + hatSwitch: number; + rawBytes: number[]; +} + +export interface FlightGamepadState { + controllerId: number; + buttons: number; + leftTrigger: number; + rightTrigger: number; + leftStickX: number; + leftStickY: number; + rightStickX: number; + rightStickY: number; + connected: boolean; +} + +export interface DiscordPresencePayload { + type: "idle" | "queue" | "streaming"; + gameName?: string; + resolution?: string; + fps?: number; + bitrateMbps?: number; + region?: string; + startTimestamp?: number; + queuePosition?: number;} diff --git a/opennow-stable/src/shared/ipc.ts b/opennow-stable/src/shared/ipc.ts index 454dc46..abb503e 100644 --- a/opennow-stable/src/shared/ipc.ts +++ b/opennow-stable/src/shared/ipc.ts @@ -27,6 +27,16 @@ export const IPC_CHANNELS = { SETTINGS_RESET: "settings:reset", LOGS_EXPORT: "logs:export", LOGS_GET_RENDERER: "logs:get-renderer", -} as const; + DISCORD_UPDATE_PRESENCE: "discord:update-presence", + DISCORD_CLEAR_PRESENCE: "discord:clear-presence", + FLIGHT_GET_PROFILE: "flight:get-profile", + FLIGHT_SET_PROFILE: "flight:set-profile", + FLIGHT_DELETE_PROFILE: "flight:delete-profile", + FLIGHT_GET_ALL_PROFILES: "flight:get-all-profiles", + FLIGHT_RESET_PROFILE: "flight:reset-profile", + HDR_GET_OS_INFO: "hdr:get-os-info", + MIC_ENUMERATE_DEVICES: "mic:enumerate-devices", + MIC_DEVICES_CHANGED: "mic:devices-changed", + APP_RELAUNCH: "app:relaunch",} as const; export type IpcChannel = (typeof IPC_CHANNELS)[keyof typeof IPC_CHANNELS]; From dc25106a90798fcafdccc35d648dd4808faa7a80 Mon Sep 17 00:00:00 2001 From: Meganugger <182369132+Meganugger@users.noreply.github.com> Date: Thu, 19 Feb 2026 22:40:02 +0000 Subject: [PATCH 2/3] Enable microphone audio transmission and relocate toggle shortcut. --- .../renderer/src/components/SettingsPage.tsx | 41 ++--- .../src/renderer/src/gfn/micAudioService.ts | 153 +++++++++++++++--- .../src/renderer/src/gfn/webrtcClient.ts | 47 +++++- 3 files changed, 195 insertions(+), 46 deletions(-) diff --git a/opennow-stable/src/renderer/src/components/SettingsPage.tsx b/opennow-stable/src/renderer/src/components/SettingsPage.tsx index 42aa0a7..a0ce1b6 100644 --- a/opennow-stable/src/renderer/src/components/SettingsPage.tsx +++ b/opennow-stable/src/renderer/src/components/SettingsPage.tsx @@ -78,7 +78,7 @@ const shortcutDefaults = { shortcutStopStream: "Ctrl+Shift+Q", shortcutToggleAntiAfk: "Ctrl+Shift+K", shortcutToggleMicrophone: "Ctrl+Shift+M", -} as const; + shortcutToggleMic: "Ctrl+Shift+M",} as const; const microphoneModeOptions: Array<{ value: MicrophoneMode; label: string }> = [ { value: "disabled", label: "Disabled" }, @@ -823,13 +823,13 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag && settings.shortcutStopStream === shortcutDefaults.shortcutStopStream && settings.shortcutToggleAntiAfk === shortcutDefaults.shortcutToggleAntiAfk && settings.shortcutToggleMicrophone === shortcutDefaults.shortcutToggleMicrophone, - [ + && settings.shortcutToggleMic === shortcutDefaults.shortcutToggleMic, [ settings.shortcutToggleStats, settings.shortcutTogglePointerLock, settings.shortcutStopStream, settings.shortcutToggleAntiAfk, settings.shortcutToggleMicrophone, - ] + settings.shortcutToggleMic, ] ); const handleResetShortcuts = useCallback(() => { @@ -838,19 +838,19 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag setStopStreamInput(shortcutDefaults.shortcutStopStream); setToggleAntiAfkInput(shortcutDefaults.shortcutToggleAntiAfk); setToggleMicrophoneInput(shortcutDefaults.shortcutToggleMicrophone); - setToggleStatsError(false); + setMicToggleShortcutInput(shortcutDefaults.shortcutToggleMic); setToggleStatsError(false); setTogglePointerLockError(false); setStopStreamError(false); setToggleAntiAfkError(false); setToggleMicrophoneError(false); - + setMicToggleShortcutError(false); const shortcutKeys = [ "shortcutToggleStats", "shortcutTogglePointerLock", "shortcutStopStream", "shortcutToggleAntiAfk", "shortcutToggleMicrophone", - ] as const; + "shortcutToggleMic", ] as const; for (const key of shortcutKeys) { const value = shortcutDefaults[key]; @@ -1432,21 +1432,6 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag
- )}
@@ -1532,7 +1517,15 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag value={toggleMicrophoneInput} onChange={(e) => setToggleMicrophoneInput(e.target.value)} onBlur={() => handleShortcutBlur("shortcutToggleMicrophone", toggleMicrophoneInput, setToggleMicrophoneInput, setToggleMicrophoneError)} - onKeyDown={handleShortcutKeyDown} + + {settings.micMode === "push-to-talk" ? "Push-to-Talk Key" : "Toggle Mic"} + + setMicToggleShortcutInput(e.target.value)} + onBlur={() => handleShortcutBlur("shortcutToggleMic", micToggleShortcutInput, setMicToggleShortcutInput, setMicToggleShortcutError)} onKeyDown={handleShortcutKeyDown} placeholder="Ctrl+Shift+M" spellCheck={false} /> @@ -1540,13 +1533,13 @@ export function SettingsPage({ settings, regions, onSettingChange }: SettingsPag
{(toggleStatsError || togglePointerLockError || stopStreamError || toggleAntiAfkError || toggleMicrophoneError) && ( - + {(toggleStatsError || togglePointerLockError || stopStreamError || toggleAntiAfkError || micToggleShortcutError) && ( Invalid shortcut. Use {shortcutExamples} )} {!toggleStatsError && !togglePointerLockError && !stopStreamError && !toggleAntiAfkError && !toggleMicrophoneError && ( - + {!toggleStatsError && !togglePointerLockError && !stopStreamError && !toggleAntiAfkError && !micToggleShortcutError && ( {shortcutExamples}. Stop: {formatShortcutForDisplay(settings.shortcutStopStream, isMac)}. Mic: {formatShortcutForDisplay(settings.shortcutToggleMicrophone, isMac)}. )} diff --git a/opennow-stable/src/renderer/src/gfn/micAudioService.ts b/opennow-stable/src/renderer/src/gfn/micAudioService.ts index 7279c23..8892cd2 100644 --- a/opennow-stable/src/renderer/src/gfn/micAudioService.ts +++ b/opennow-stable/src/renderer/src/gfn/micAudioService.ts @@ -11,6 +11,7 @@ export interface MicAudioState { level: number; deviceId: string; deviceLabel: string; + micTxActive: boolean; } export type MicStateListener = (state: MicAudioState) => void; @@ -46,6 +47,9 @@ export class MicAudioService { private rtcSender: RTCRtpSender | null = null; private peerConnection: RTCPeerConnection | null = null; + private micTxActive = false; + private lastMicBytesSent = 0; + private micTxPollId: ReturnType | null = null; getState(): MicAudioState { return { @@ -53,6 +57,7 @@ export class MicAudioService { level: this.currentLevel, deviceId: this.deviceId, deviceLabel: this.currentDeviceLabel, + micTxActive: this.micTxActive, }; } @@ -256,12 +261,12 @@ export class MicAudioService { stopCapture(): void { this.stopLevelMonitor(); + this.stopMicTxPoll(); - if (this.rtcSender && this.peerConnection) { + if (this.rtcSender) { try { - this.peerConnection.removeTrack(this.rtcSender); + void this.rtcSender.replaceTrack(null); } catch { /* ignore */ } - this.rtcSender = null; } if (this.sourceNode) { @@ -300,12 +305,13 @@ export class MicAudioService { } setPeerConnection(pc: RTCPeerConnection | null): void { - if (this.rtcSender && this.peerConnection) { + if (this.rtcSender) { try { - this.peerConnection.removeTrack(this.rtcSender); + void this.rtcSender.replaceTrack(null); } catch { /* ignore */ } this.rtcSender = null; } + this.stopMicTxPoll(); this.peerConnection = pc; @@ -314,35 +320,142 @@ export class MicAudioService { } } + bindMicTransceiver(): void { + if (this.peerConnection && this.outputTrack && !this.rtcSender) { + this.attachToPeerConnection(); + } + } + private attachToPeerConnection(): void { if (!this.peerConnection || !this.outputTrack) return; if (this.rtcSender) { try { void this.rtcSender.replaceTrack(this.outputTrack); + console.log("[Mic] Replaced track on existing sender"); + this.startMicTxPoll(); return; } catch { /* ignore */ } } - try { - this.rtcSender = this.peerConnection.addTrack( - this.outputTrack, - this.destinationNode!.stream, - ); + const transceivers = this.peerConnection.getTransceivers(); + this.logTransceiverState(transceivers); + + if (transceivers.length === 0) { + console.log("[Mic] No transceivers available yet (SDP not negotiated)"); + return; + } + + let micTransceiver: RTCRtpTransceiver | null = null; + + for (const t of transceivers) { + const kind = t.receiver.track?.kind ?? t.sender.track?.kind; + if (kind === "audio" && (t.direction === "sendonly" || t.direction === "sendrecv")) { + micTransceiver = t; + console.log(`[Mic] Found mic transceiver by direction: mid=${t.mid}, direction=${t.direction}`); + break; + } + } - if (this.rtcSender) { - const params = this.rtcSender.getParameters(); - if (!params.encodings || params.encodings.length === 0) { - params.encodings = [{}]; + if (!micTransceiver && transceivers.length > 2) { + const t = transceivers[2]; + const kind = t.receiver.track?.kind ?? t.sender.track?.kind; + if (kind === "audio") { + micTransceiver = t; + console.log(`[Mic] Found mic transceiver by index 2 fallback: mid=${t.mid}, direction=${t.direction}`); + } + } + + if (!micTransceiver) { + for (const t of transceivers) { + const kind = t.receiver.track?.kind ?? t.sender.track?.kind; + if (kind === "audio" && !t.sender.track && t.direction !== "recvonly") { + micTransceiver = t; + console.log(`[Mic] Found mic transceiver by no-track fallback: mid=${t.mid}, direction=${t.direction}`); + break; } - params.encodings[0].maxBitrate = 64_000; - params.encodings[0].networkPriority = "high"; - params.encodings[0].priority = "high"; - void this.rtcSender.setParameters(params).catch(() => { /* ignore */ }); } + } + + if (!micTransceiver) { + console.warn("[Mic] Could not find mic transceiver - mic audio will NOT transmit upstream"); + return; + } + + try { + void micTransceiver.sender.replaceTrack(this.outputTrack); + this.rtcSender = micTransceiver.sender; + + const params = this.rtcSender.getParameters(); + if (!params.encodings || params.encodings.length === 0) { + params.encodings = [{}]; + } + params.encodings[0].maxBitrate = 64_000; + params.encodings[0].networkPriority = "high"; + params.encodings[0].priority = "high"; + void this.rtcSender.setParameters(params).catch(() => { /* ignore */ }); + + console.log(`[Mic] Track attached to negotiated transceiver mid=${micTransceiver.mid}, direction=${micTransceiver.direction}`); + this.startMicTxPoll(); } catch (err) { - console.error("[Mic] Failed to attach track to peer connection:", err); + console.error("[Mic] Failed to attach track to mic transceiver:", err); + } + } + + private logTransceiverState(transceivers: RTCRtpTransceiver[]): void { + console.log(`[Mic] === Transceiver diagnostics (${transceivers.length} transceivers) ===`); + for (let i = 0; i < transceivers.length; i++) { + const t = transceivers[i]; + console.log( + `[Mic] [${i}] mid=${t.mid} direction=${t.direction} currentDirection=${t.currentDirection}` + + ` sender.track=${t.sender.track ? `${t.sender.track.kind}(enabled=${t.sender.track.enabled})` : "null"}` + + ` receiver.track=${t.receiver.track ? `${t.receiver.track.kind}(enabled=${t.receiver.track.enabled})` : "null"}` + ); + } + } + + private startMicTxPoll(): void { + this.stopMicTxPoll(); + if (!this.peerConnection || !this.rtcSender) return; + + this.micTxPollId = setInterval(() => { + void this.pollMicTxStats(); + }, 1000); + } + + private stopMicTxPoll(): void { + if (this.micTxPollId !== null) { + clearInterval(this.micTxPollId); + this.micTxPollId = null; } + this.micTxActive = false; + this.lastMicBytesSent = 0; + } + + private async pollMicTxStats(): Promise { + if (!this.peerConnection || !this.rtcSender) return; + + try { + const report = await this.peerConnection.getStats(this.rtcSender); + for (const entry of report.values()) { + const stats = entry as unknown as Record; + if (entry.type === "outbound-rtp" && stats.kind === "audio") { + const bytesSent = Number(stats.bytesSent ?? 0); + const packetsSent = Number(stats.packetsSent ?? 0); + const mid = String(stats.mid ?? ""); + const wasActive = this.micTxActive; + this.micTxActive = bytesSent > this.lastMicBytesSent; + + if (this.micTxActive !== wasActive) { + console.log(`[Mic] TX ${this.micTxActive ? "Active" : "Idle"} — bytesSent=${bytesSent} packetsSent=${packetsSent} mid=${mid}`); + this.emit(); + } + + this.lastMicBytesSent = bytesSent; + break; + } + } + } catch { /* ignore stats errors */ } } private startLevelMonitor(): void { @@ -442,8 +555,10 @@ export class MicAudioService { dispose(): void { this.stopCapture(); + this.stopMicTxPoll(); this.removeDeviceChangeListener(); this.listeners.clear(); + this.rtcSender = null; this.peerConnection = null; } } diff --git a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts index 375f12d..ff74c2a 100644 --- a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts +++ b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts @@ -164,7 +164,12 @@ export interface StreamDiagnostics { inputQueueDropCount: number; inputQueueMaxSchedulingDelayMs: number; - // System info + // HDR diagnostics + hdrState: HdrStreamState; + + // Mic TX stats + micBytesSent: number; + micPacketsSent: number; // System info gpuType: string; serverRegion: string; @@ -550,7 +555,18 @@ export class GfnWebRtcClient { inputQueuePeakBufferedBytes: 0, inputQueueDropCount: 0, inputQueueMaxSchedulingDelayMs: 0, - gpuType: "", + micBytesSent: 0, + micPacketsSent: 0, + hdrState: { + status: "inactive", + bitDepth: 8, + colorPrimaries: "BT.709", + transferFunction: "SDR", + matrixCoefficients: "BT.709", + codecProfile: "", + overlayForcesSdr: false, + fallbackReason: null, + }, gpuType: "", serverRegion: "", micState: "uninitialized", micEnabled: false, @@ -686,7 +702,18 @@ export class GfnWebRtcClient { inputQueuePeakBufferedBytes: 0, inputQueueDropCount: 0, inputQueueMaxSchedulingDelayMs: 0, - gpuType: this.gpuType, + micBytesSent: 0, + micPacketsSent: 0, + hdrState: { + status: "inactive", + bitDepth: 8, + colorPrimaries: "BT.709", + transferFunction: "SDR", + matrixCoefficients: "BT.709", + codecProfile: "", + overlayForcesSdr: false, + fallbackReason: null, + }, gpuType: this.gpuType, serverRegion: this.serverRegion, micState: this.micState, micEnabled: this.micManager?.isEnabled() ?? false, @@ -962,6 +989,15 @@ export class GfnWebRtcClient { this.inputQueuePeakBufferedBytesWindow = reliableBufferedAmount; this.inputQueueMaxSchedulingDelayMsWindow = 0; + for (const entry of report.values()) { + const stats = entry as unknown as Record; + if (entry.type === "outbound-rtp" && stats.kind === "audio") { + this.diagnostics.micBytesSent = Number(stats.bytesSent ?? 0); + this.diagnostics.micPacketsSent = Number(stats.packetsSent ?? 0); + break; + } + } + this.emitStats(); } @@ -2658,6 +2694,11 @@ export class GfnWebRtcClient { // but before createAnswer (which generates the answer SDP). this.applyCodecPreferences(pc, effectiveCodec, preferredHevcProfileId); + if (this.micService) { + this.log("Binding mic transceiver after setRemoteDescription"); + this.micService.bindMicTransceiver(); + } + // 4. Create answer, munge SDP, and set local description this.log("Creating answer..."); const answer = await pc.createAnswer(); From fae866fb12e5c04ad444708334b5b8b69218065b Mon Sep 17 00:00:00 2001 From: Meganugger <182369132+Meganugger@users.noreply.github.com> Date: Thu, 19 Feb 2026 23:03:39 +0000 Subject: [PATCH 3/3] Enable mic audio transmission by fixing Electron permissions and WebRTC transceiver binding. --- opennow-stable/src/main/index.ts | 23 ++- .../src/renderer/src/gfn/micAudioService.ts | 105 ++++--------- .../src/renderer/src/gfn/webrtcClient.ts | 138 +++++++++++++++++- 3 files changed, 185 insertions(+), 81 deletions(-) diff --git a/opennow-stable/src/main/index.ts b/opennow-stable/src/main/index.ts index a4adca9..565471c 100644 --- a/opennow-stable/src/main/index.ts +++ b/opennow-stable/src/main/index.ts @@ -1,5 +1,5 @@ import { app, BrowserWindow, ipcMain, dialog, systemPreferences, session } from "electron"; -import { fileURLToPath } from "node:url"; +import { app, BrowserWindow, ipcMain, dialog, session, systemPreferences } from "electron";import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { existsSync, readFileSync } from "node:fs"; @@ -202,12 +202,20 @@ function setupWebHidPermissions(): void { }); ses.setPermissionCheckHandler((_webContents, permission) => { - if (permission === "hid" || permission === "media") { + if (permission === "hid" || permission === "media" || permission === "keyboardLock") { return true; } return true; }); + ses.setPermissionRequestHandler((_webContents, permission, callback) => { + if (permission === "media" || permission === "keyboardLock") { + callback(true); + return; + } + callback(true); + }); + ses.on("select-hid-device", (event, details, callback) => { event.preventDefault(); const ungranted = details.deviceList.find((d) => !grantedHidDeviceIds.has(d.deviceId)); @@ -643,7 +651,16 @@ app.whenReady().then(async () => { return allowedPermissions.has(permission); }); - registerIpcHandlers(); + if (process.platform === "darwin") { + const micAccess = systemPreferences.getMediaAccessStatus("microphone"); + console.log(`[Main] macOS microphone access status: ${micAccess}`); + if (micAccess !== "granted") { + const granted = await systemPreferences.askForMediaAccess("microphone"); + console.log(`[Main] macOS microphone access prompt result: ${granted}`); + } + } + + setupWebHidPermissions(); registerIpcHandlers(); await createMainWindow(); app.on("activate", async () => { diff --git a/opennow-stable/src/renderer/src/gfn/micAudioService.ts b/opennow-stable/src/renderer/src/gfn/micAudioService.ts index 8892cd2..eea64b6 100644 --- a/opennow-stable/src/renderer/src/gfn/micAudioService.ts +++ b/opennow-stable/src/renderer/src/gfn/micAudioService.ts @@ -47,6 +47,7 @@ export class MicAudioService { private rtcSender: RTCRtpSender | null = null; private peerConnection: RTCPeerConnection | null = null; + private micTransceiver: RTCRtpTransceiver | null = null; private micTxActive = false; private lastMicBytesSent = 0; private micTxPollId: ReturnType | null = null; @@ -231,7 +232,7 @@ export class MicAudioService { if (this.outputTrack) { this.updateMuteState(); - this.attachToPeerConnection(); + this.attachTrackToTransceiver(); } this.startLevelMonitor(); @@ -312,79 +313,36 @@ export class MicAudioService { this.rtcSender = null; } this.stopMicTxPoll(); + this.micTransceiver = null; this.peerConnection = pc; - - if (pc && this.outputTrack) { - this.attachToPeerConnection(); - } } - bindMicTransceiver(): void { - if (this.peerConnection && this.outputTrack && !this.rtcSender) { - this.attachToPeerConnection(); - } - } + setMicTransceiver(transceiver: RTCRtpTransceiver): void { + this.micTransceiver = transceiver; + this.rtcSender = transceiver.sender; - private attachToPeerConnection(): void { - if (!this.peerConnection || !this.outputTrack) return; + console.log( + `[Mic] Transceiver assigned: mid=${transceiver.mid} direction=${transceiver.direction}` + + ` currentDirection=${transceiver.currentDirection}` + + ` sender.track=${transceiver.sender.track?.kind ?? "null"}`, + ); - if (this.rtcSender) { - try { - void this.rtcSender.replaceTrack(this.outputTrack); - console.log("[Mic] Replaced track on existing sender"); - this.startMicTxPoll(); - return; - } catch { /* ignore */ } - } - - const transceivers = this.peerConnection.getTransceivers(); - this.logTransceiverState(transceivers); - - if (transceivers.length === 0) { - console.log("[Mic] No transceivers available yet (SDP not negotiated)"); - return; - } - - let micTransceiver: RTCRtpTransceiver | null = null; - - for (const t of transceivers) { - const kind = t.receiver.track?.kind ?? t.sender.track?.kind; - if (kind === "audio" && (t.direction === "sendonly" || t.direction === "sendrecv")) { - micTransceiver = t; - console.log(`[Mic] Found mic transceiver by direction: mid=${t.mid}, direction=${t.direction}`); - break; - } - } - - if (!micTransceiver && transceivers.length > 2) { - const t = transceivers[2]; - const kind = t.receiver.track?.kind ?? t.sender.track?.kind; - if (kind === "audio") { - micTransceiver = t; - console.log(`[Mic] Found mic transceiver by index 2 fallback: mid=${t.mid}, direction=${t.direction}`); - } + if (this.outputTrack) { + this.attachTrackToTransceiver(); } + } - if (!micTransceiver) { - for (const t of transceivers) { - const kind = t.receiver.track?.kind ?? t.sender.track?.kind; - if (kind === "audio" && !t.sender.track && t.direction !== "recvonly") { - micTransceiver = t; - console.log(`[Mic] Found mic transceiver by no-track fallback: mid=${t.mid}, direction=${t.direction}`); - break; - } - } - } + private attachTrackToTransceiver(): void { + if (!this.outputTrack) return; - if (!micTransceiver) { - console.warn("[Mic] Could not find mic transceiver - mic audio will NOT transmit upstream"); + if (!this.micTransceiver || !this.rtcSender) { + console.log("[Mic] No mic transceiver assigned yet — track will attach when transceiver is provided"); return; } try { - void micTransceiver.sender.replaceTrack(this.outputTrack); - this.rtcSender = micTransceiver.sender; + void this.rtcSender.replaceTrack(this.outputTrack); const params = this.rtcSender.getParameters(); if (!params.encodings || params.encodings.length === 0) { @@ -395,25 +353,18 @@ export class MicAudioService { params.encodings[0].priority = "high"; void this.rtcSender.setParameters(params).catch(() => { /* ignore */ }); - console.log(`[Mic] Track attached to negotiated transceiver mid=${micTransceiver.mid}, direction=${micTransceiver.direction}`); + console.log( + `[Mic] Track attached to transceiver mid=${this.micTransceiver.mid}` + + ` direction=${this.micTransceiver.direction}` + + ` track.enabled=${this.outputTrack.enabled}`, + ); + this.startMicTxPoll(); } catch (err) { console.error("[Mic] Failed to attach track to mic transceiver:", err); } } - private logTransceiverState(transceivers: RTCRtpTransceiver[]): void { - console.log(`[Mic] === Transceiver diagnostics (${transceivers.length} transceivers) ===`); - for (let i = 0; i < transceivers.length; i++) { - const t = transceivers[i]; - console.log( - `[Mic] [${i}] mid=${t.mid} direction=${t.direction} currentDirection=${t.currentDirection}` + - ` sender.track=${t.sender.track ? `${t.sender.track.kind}(enabled=${t.sender.track.enabled})` : "null"}` + - ` receiver.track=${t.receiver.track ? `${t.receiver.track.kind}(enabled=${t.receiver.track.enabled})` : "null"}` - ); - } - } - private startMicTxPoll(): void { this.stopMicTxPoll(); if (!this.peerConnection || !this.rtcSender) return; @@ -447,7 +398,10 @@ export class MicAudioService { this.micTxActive = bytesSent > this.lastMicBytesSent; if (this.micTxActive !== wasActive) { - console.log(`[Mic] TX ${this.micTxActive ? "Active" : "Idle"} — bytesSent=${bytesSent} packetsSent=${packetsSent} mid=${mid}`); + console.log( + `[Mic] TX ${this.micTxActive ? "Active" : "Idle"}` + + ` — bytesSent=${bytesSent} packetsSent=${packetsSent} mid=${mid}`, + ); this.emit(); } @@ -559,6 +513,7 @@ export class MicAudioService { this.removeDeviceChangeListener(); this.listeners.clear(); this.rtcSender = null; + this.micTransceiver = null; this.peerConnection = null; } } diff --git a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts index ff74c2a..25e1b27 100644 --- a/opennow-stable/src/renderer/src/gfn/webrtcClient.ts +++ b/opennow-stable/src/renderer/src/gfn/webrtcClient.ts @@ -511,6 +511,7 @@ export class GfnWebRtcClient { private pendingMouseTimestampUs: bigint | null = null; private mouseDeltaFilter = new MouseDeltaFilter(); private micService: MicAudioService | null = null; + private micTransceiver: RTCRtpTransceiver | null = null; private partialReliableThresholdMs = GfnWebRtcClient.DEFAULT_PARTIAL_RELIABLE_THRESHOLD_MS; private inputQueuePeakBufferedBytesWindow = 0; @@ -992,8 +993,15 @@ export class GfnWebRtcClient { for (const entry of report.values()) { const stats = entry as unknown as Record; if (entry.type === "outbound-rtp" && stats.kind === "audio") { - this.diagnostics.micBytesSent = Number(stats.bytesSent ?? 0); - this.diagnostics.micPacketsSent = Number(stats.packetsSent ?? 0); + const bytesSent = Number(stats.bytesSent ?? 0); + const packetsSent = Number(stats.packetsSent ?? 0); + const mid = String(stats.mid ?? ""); + const prevBytes = this.diagnostics.micBytesSent; + this.diagnostics.micBytesSent = bytesSent; + this.diagnostics.micPacketsSent = packetsSent; + if (bytesSent > 0 && bytesSent !== prevBytes) { + this.log(`Outbound audio RTP: mid=${mid} bytesSent=${bytesSent} packetsSent=${packetsSent}`); + } break; } } @@ -2459,7 +2467,129 @@ export class GfnWebRtcClient { this.micService = service; if (service) { service.setPeerConnection(this.pc); + if (this.micTransceiver) { + service.setMicTransceiver(this.micTransceiver); + } + } + } + + private bindMicTransceiverFromOffer(pc: RTCPeerConnection, offerSdp: string): void { + const transceivers = pc.getTransceivers(); + + this.log(`=== Mic transceiver binding: ${transceivers.length} transceivers ===`); + for (let i = 0; i < transceivers.length; i++) { + const t = transceivers[i]; + this.log( + ` [${i}] mid=${t.mid} direction=${t.direction} currentDirection=${t.currentDirection}` + + ` sender.track=${t.sender.track ? `${t.sender.track.kind}(enabled=${t.sender.track.enabled})` : "null"}` + + ` receiver.track=${t.receiver.track ? t.receiver.track.kind : "null"}`, + ); + } + + let micT: RTCRtpTransceiver | null = null; + + const mid3 = transceivers.find( + (t) => t.mid === "3" && (t.receiver.track?.kind === "audio" || t.sender.track?.kind === "audio"), + ); + if (mid3) { + micT = mid3; + this.log(`Mic transceiver: matched mid=3 (direction=${mid3.direction})`); + } + + if (!micT) { + const micMid = this.deriveMicMidFromOfferSdp(offerSdp); + if (micMid !== null) { + const byMid = transceivers.find( + (t) => t.mid === micMid && (t.receiver.track?.kind === "audio" || t.sender.track?.kind === "audio"), + ); + if (byMid) { + micT = byMid; + this.log(`Mic transceiver: matched via SDP m-line mapping mid=${micMid} (direction=${byMid.direction})`); + } else { + this.log(`Mic transceiver: SDP says mid=${micMid} but no matching audio transceiver found`); + } + } else { + this.log("Mic transceiver: could not derive mic mid from offer SDP"); + } + } + + if (!micT) { + this.log("WARNING: No mic transceiver found — mic audio will NOT transmit upstream"); + return; + } + + if (micT.direction === "recvonly" || micT.direction === "inactive") { + const wasDirec = micT.direction; + try { + micT.direction = "sendrecv"; + this.log(`Mic transceiver direction changed: ${wasDirec} → sendrecv`); + } catch (err) { + this.log(`Failed to change mic transceiver direction: ${String(err)}`); + } + } + + this.micTransceiver = micT; + + if (this.micService) { + this.micService.setMicTransceiver(micT); + } + } + + private deriveMicMidFromOfferSdp(sdp: string): string | null { + const lines = sdp.split(/\r?\n/); + let mLineIndex = -1; + let currentMid: string | null = null; + let currentType = ""; + let currentDirection = ""; + const mLineSections: { index: number; type: string; mid: string | null; direction: string }[] = []; + + for (const line of lines) { + if (line.startsWith("m=")) { + if (mLineIndex >= 0) { + mLineSections.push({ index: mLineIndex, type: currentType, mid: currentMid, direction: currentDirection }); + } + mLineIndex++; + currentType = line.split(" ")[0].replace("m=", ""); + currentMid = null; + currentDirection = ""; + } else if (line.startsWith("a=mid:")) { + currentMid = line.replace("a=mid:", "").trim(); + } else if ( + line === "a=recvonly" || line === "a=sendonly" || + line === "a=sendrecv" || line === "a=inactive" + ) { + currentDirection = line.replace("a=", ""); + } + } + if (mLineIndex >= 0) { + mLineSections.push({ index: mLineIndex, type: currentType, mid: currentMid, direction: currentDirection }); + } + + this.log(`Offer SDP m-line map: ${mLineSections.map((s) => `[${s.index}] m=${s.type} mid=${s.mid} dir=${s.direction}`).join(" | ")}`); + + if (mLineSections.length >= 4) { + const micSection = mLineSections[3]; + if (micSection.mid) { + this.log(`SDP fallback: m-line index 3 (m=${micSection.type}) has mid=${micSection.mid}`); + return micSection.mid; + } } + + const audioSections = mLineSections.filter((s) => s.type === "audio"); + if (audioSections.length >= 2) { + const micCandidate = audioSections.find((s) => s.direction === "recvonly" || s.direction === "inactive"); + if (micCandidate?.mid) { + this.log(`SDP fallback: recvonly/inactive audio m-line at index ${micCandidate.index} has mid=${micCandidate.mid}`); + return micCandidate.mid; + } + const second = audioSections[1]; + if (second?.mid) { + this.log(`SDP fallback: second audio m-line at index ${second.index} has mid=${second.mid}`); + return second.mid; + } + } + + return null; } getPeerConnection(): RTCPeerConnection | null { @@ -2525,6 +2655,7 @@ export class GfnWebRtcClient { if (this.micService) { this.micService.setPeerConnection(pc); } + this.micTransceiver = null; pc.onicecandidate = (event) => { if (!event.candidate) { @@ -2696,8 +2827,8 @@ export class GfnWebRtcClient { if (this.micService) { this.log("Binding mic transceiver after setRemoteDescription"); - this.micService.bindMicTransceiver(); } + this.bindMicTransceiverFromOffer(pc, filteredOffer); // 4. Create answer, munge SDP, and set local description this.log("Creating answer..."); @@ -2834,6 +2965,7 @@ export class GfnWebRtcClient { if (this.micService) { this.micService.setPeerConnection(null); } + this.micTransceiver = null; this.cleanupPeerConnection();