From 25dba140b6f0b5bdbb3eeef5970b0cf27b1b0648 Mon Sep 17 00:00:00 2001 From: Logan Abell Date: Tue, 17 Mar 2026 20:48:55 -0400 Subject: [PATCH 1/8] first working version --- README.md | 5 +- cmake/compile_definitions/windows.cmake | 4 + cmake/dependencies/Boost_Sunshine.cmake | 5 +- cmake/packaging/windows.cmake | 8 + cmake/targets/common.cmake | 13 +- docs/configuration.md | 79 ++++ docs/remote_microphone.md | 69 +++ src/audio.cpp | 253 ++++++++++ src/audio.h | 52 +++ src/config.cpp | 6 + src/config.h | 3 + src/confighttp.cpp | 49 ++ src/crypto.cpp | 37 ++ src/crypto.h | 1 + src/platform/common.h | 5 + src/platform/linux/audio.cpp | 105 +++++ src/platform/macos/microphone.mm | 15 + src/platform/windows/apollo_vmic.cpp | 60 +++ src/platform/windows/apollo_vmic.h | 38 ++ src/platform/windows/audio.cpp | 99 +++- src/platform/windows/mic_write.cpp | 439 ++++++++++++++++++ src/platform/windows/mic_write.h | 56 +++ src/rtsp.cpp | 22 + src/rtsp.h | 1 + src/stream.cpp | 163 ++++++- src/stream.h | 1 + .../common/assets/web/Troubleshooting.vue | 269 +++++++++++ .../assets/web/configs/tabs/AudioVideo.vue | 11 +- .../assets/web/public/assets/locale/en.json | 15 +- .../web/public/assets/locale/en_GB.json | 13 + .../web/public/assets/locale/en_US.json | 13 + .../windows/drivers/apollo-vmic/LICENSE.txt | 21 + .../windows/drivers/apollo-vmic/README.txt | 18 + .../apollo-vmic/THIRD_PARTY_NOTICES.md | 33 ++ .../apollo-vmic/VirtualAudioDriver.inf | Bin 0 -> 14822 bytes .../apollo-vmic/VirtualAudioDriver.sys | Bin 0 -> 74552 bytes .../windows/drivers/apollo-vmic/install.bat | 22 + .../windows/drivers/apollo-vmic/uninstall.bat | 16 + .../apollo-vmic/virtualaudiodriver.cat | Bin 0 -> 14122 bytes src_assets/windows/drivers/vbcable/NOTICE.txt | 9 + .../windows/drivers/vbcable/install.bat | 58 +++ .../windows/drivers/vbcable/uninstall.bat | 35 ++ third-party/moonlight-common-c | 2 +- tools/CMakeLists.txt | 14 + tools/apollovmicctl.cpp | 290 ++++++++++++ 45 files changed, 2398 insertions(+), 29 deletions(-) create mode 100644 docs/remote_microphone.md create mode 100644 src/platform/windows/apollo_vmic.cpp create mode 100644 src/platform/windows/apollo_vmic.h create mode 100644 src/platform/windows/mic_write.cpp create mode 100644 src/platform/windows/mic_write.h create mode 100644 src_assets/windows/drivers/apollo-vmic/LICENSE.txt create mode 100644 src_assets/windows/drivers/apollo-vmic/README.txt create mode 100644 src_assets/windows/drivers/apollo-vmic/THIRD_PARTY_NOTICES.md create mode 100644 src_assets/windows/drivers/apollo-vmic/VirtualAudioDriver.inf create mode 100644 src_assets/windows/drivers/apollo-vmic/VirtualAudioDriver.sys create mode 100644 src_assets/windows/drivers/apollo-vmic/install.bat create mode 100644 src_assets/windows/drivers/apollo-vmic/uninstall.bat create mode 100644 src_assets/windows/drivers/apollo-vmic/virtualaudiodriver.cat create mode 100644 src_assets/windows/drivers/vbcable/NOTICE.txt create mode 100644 src_assets/windows/drivers/vbcable/install.bat create mode 100644 src_assets/windows/drivers/vbcable/uninstall.bat create mode 100644 tools/apollovmicctl.cpp diff --git a/README.md b/README.md index 904abb205..d2979fc6d 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ Vibepollo is an AI‑enhanced version of Apollo, a popular remote streaming application. It intends to integrate all scripts from myself (Nonary) and more. - - ## Key Features * **Display Setting Automation** @@ -35,6 +33,9 @@ Vibepollo is an AI‑enhanced version of Apollo, a popular remote streaming appl * **Lossless Scaling & NVIDIA Smooth Motion** Vibepollo can automatically apply optimal Lossless Scaling settings to generate frames for any application. On RTX 40‑series and newer GPUs, you can optionally enable **NVIDIA Smooth Motion** for better performance and image quality (while Lossless Scaling remains more customizable). +* **Remote Microphone Passthrough** + Accept redirected client microphone audio from compatible Moonlight or Artemis builds, decode it on the host, and render it into VB-CABLE on Windows so host apps can use `CABLE Output` as their microphone source. Setup and debugging notes are in [docs/remote_microphone.md](docs/remote_microphone.md). + * **API Token Management** Access tokens can be tightly scoped—down to specific methods—so external scripts don’t need full administrative rights. This improves security while keeping automation flexible. diff --git a/cmake/compile_definitions/windows.cmake b/cmake/compile_definitions/windows.cmake index 5bbddb760..4f5806535 100644 --- a/cmake/compile_definitions/windows.cmake +++ b/cmake/compile_definitions/windows.cmake @@ -180,6 +180,10 @@ set(PLATFORM_TARGET_FILES "${CMAKE_SOURCE_DIR}/src/platform/windows/display_vram.cpp" "${CMAKE_SOURCE_DIR}/src/platform/windows/display_wgc.cpp" "${CMAKE_SOURCE_DIR}/src/platform/windows/audio.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/windows/apollo_vmic.h" + "${CMAKE_SOURCE_DIR}/src/platform/windows/apollo_vmic.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/windows/mic_write.h" + "${CMAKE_SOURCE_DIR}/src/platform/windows/mic_write.cpp" "${CMAKE_SOURCE_DIR}/src/platform/windows/virtual_display.h" "${CMAKE_SOURCE_DIR}/src/platform/windows/virtual_display.cpp" "${CMAKE_SOURCE_DIR}/src/platform/windows/virtual_display_legacy.h" diff --git a/cmake/dependencies/Boost_Sunshine.cmake b/cmake/dependencies/Boost_Sunshine.cmake index 1ea68c02e..ad5a39c23 100644 --- a/cmake/dependencies/Boost_Sunshine.cmake +++ b/cmake/dependencies/Boost_Sunshine.cmake @@ -3,7 +3,8 @@ # include_guard(GLOBAL) -set(BOOST_VERSION "1.89.0") +set(BOOST_MIN_VERSION "1.89.0") +set(BOOST_FETCH_VERSION "1.89.0") set(BOOST_COMPONENTS filesystem locale @@ -34,7 +35,7 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.30") endif() find_package(Boost CONFIG ${BOOST_VERSION} COMPONENTS ${BOOST_COMPONENTS}) if(NOT Boost_FOUND) - message(STATUS "Boost v${BOOST_VERSION} package not found in the system. Falling back to FetchContent.") + message(STATUS "Boost v${BOOST_MIN_VERSION}+ package not found in the system. Falling back to FetchContent.") include(FetchContent) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") diff --git a/cmake/packaging/windows.cmake b/cmake/packaging/windows.cmake index 04dd477d3..5168cb7b4 100644 --- a/cmake/packaging/windows.cmake +++ b/cmake/packaging/windows.cmake @@ -22,6 +22,7 @@ endif() # Adding tools install(TARGETS dxgi-info RUNTIME DESTINATION "tools" COMPONENT dxgi) install(TARGETS audio-info RUNTIME DESTINATION "tools" COMPONENT audio) +install(TARGETS apollovmicctl RUNTIME DESTINATION "tools" COMPONENT application) # Helpers and tools # - Playnite launcher helper used for Playnite-managed app launches @@ -64,6 +65,9 @@ unset(_sudovda_file_size) install(FILES ${SUDOVDA_DRIVER_FILES} DESTINATION "drivers/sudovda" COMPONENT sudovda) +install(DIRECTORY "${SUNSHINE_SOURCE_ASSETS_DIR}/windows/drivers/vbcable" + DESTINATION "drivers" + COMPONENT vbcable) # Mandatory scripts install(DIRECTORY "${SUNSHINE_SOURCE_ASSETS_DIR}/windows/misc/service/" @@ -149,6 +153,10 @@ set(CPACK_COMPONENT_SUDOVDA_DESCRIPTION "Driver required for Virtual Display to set(CPACK_COMPONENT_SUDOVDA_GROUP "Drivers") set(CPACK_COMPONENT_SUDOVDA_REQUIRED true) +set(CPACK_COMPONENT_VBCABLE_DISPLAY_NAME "VB-CABLE") +set(CPACK_COMPONENT_VBCABLE_DESCRIPTION "Official VB-CABLE dependency for Apollo microphone passthrough on Windows.") +set(CPACK_COMPONENT_VBCABLE_GROUP "Drivers") + # audio tool set(CPACK_COMPONENT_AUDIO_DISPLAY_NAME "audio-info") set(CPACK_COMPONENT_AUDIO_DESCRIPTION "CLI tool providing information about sound devices.") diff --git a/cmake/targets/common.cmake b/cmake/targets/common.cmake index 3d455ffad..634f43325 100644 --- a/cmake/targets/common.cmake +++ b/cmake/targets/common.cmake @@ -58,7 +58,18 @@ endif() set(WEB_UI_DIR "${SUNSHINE_SOURCE_ASSETS_DIR}/common/assets/web") #WebUI build -find_program(NPM npm REQUIRED) +if(WIN32) + unset(NODE CACHE) + unset(NPM_CLI CACHE) + find_program(NODE node.exe REQUIRED) + find_file(NPM_CLI npm-cli.js + PATHS + "${CMAKE_PREFIX_PATH}" + "C:/msys64/ucrt64/lib/node_modules/npm/bin" + REQUIRED) +else() + find_program(NPM npm REQUIRED) +endif() set(NPM_INSTALL_FLAGS --ignore-scripts diff --git a/docs/configuration.md b/docs/configuration.md index 0669a9a59..ad92287cb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -851,6 +851,85 @@ editing the `conf` file in a text editor. Use the examples as reference. +### mic_backend + + + + + + + + + + + + + + +
Description + Select how Vibepollo exposes redirected client microphone audio on Windows. + In this fork, Windows microphone redirection is standardized on the `vb_cable` backend. + Vibepollo renders decoded client microphone audio into the VB-CABLE playback endpoint, and host applications + should select the paired `CABLE Output` recording device. + @note{This option is currently only used on Windows hosts.} +
Default@code{} + vb_cable + @endcode
Example@code{} + mic_backend = vb_cable + @endcode
+ +### mic_device + + + + + + + + + + + + + + + + + + +
Description + The host-side device used for redirected client microphone audio. + On Windows, Vibepollo currently auto-detects the VB-CABLE render endpoint and this value is typically left unset. + On Linux and macOS this should point at the virtual device Vibepollo writes into. +
DefaultUnset.
Example (Windows)@code{} + mic_device = CABLE Input (VB-Audio Virtual Cable) + @endcode
Example (Linux)@code{} + mic_device = sunshine-mic + @endcode
+ +### stream_mic + + + + + + + + + + + + + + +
Description + Whether Vibepollo should accept redirected client microphone audio and inject it into a host-side microphone backend. +
Default@code{} + disabled + @endcode
Example@code{} + stream_mic = enabled + @endcode
+ ### install_steam_audio_drivers diff --git a/docs/remote_microphone.md b/docs/remote_microphone.md new file mode 100644 index 000000000..45077bf0d --- /dev/null +++ b/docs/remote_microphone.md @@ -0,0 +1,69 @@ +# Remote Microphone Support + +This feature adds a working host-side remote microphone path for Vibepollo, focused on Windows hosts and VB-CABLE integration. + +## Overview + +The microphone path is: + +1. A compatible Moonlight or Artemis client captures local microphone audio. +2. The client sends encrypted or unencrypted microphone packets to Vibepollo on the dedicated microphone stream. +3. Vibepollo receives the packets, decrypts them when needed, and decodes the Opus frames on the host. +4. Vibepollo renders the decoded PCM into the VB-CABLE playback endpoint `CABLE Input`. +5. Host applications consume that audio from the VB-CABLE recording endpoint `CABLE Output`. + +This keeps the host-side application flow simple: Vibepollo writes into VB-CABLE, and games, chat apps, or capture tools use `CABLE Output` as the microphone. + +## What Changed + +The working implementation includes: + +- Dedicated microphone session handling in the stream path, including packet receive, optional decryption, and per-session lifecycle management. +- Windows microphone backend initialization and teardown that stays alive for the full remote microphone session. +- A VB-CABLE-backed Windows render path that auto-detects `CABLE Input`, creates a shared-mode WASAPI render client, decodes Opus microphone frames, and writes PCM into the render buffer. +- Host-side recovery for recoverable WASAPI failures such as device invalidation or audio service restarts. +- A Remote Microphone Debug panel in the troubleshooting UI that shows packet arrival, decode status, render status, signal detection, counters, and recent mic events. + +## Key Files + +- `src/stream.cpp`: microphone socket handling, session startup/shutdown, and packet routing. +- `src/audio.cpp`: shared microphone debug state and persistent audio context ownership for the redirect device. +- `src/platform/windows/audio.cpp`: Windows microphone backend selection and redirect device ownership. +- `src/platform/windows/apollo_vmic.cpp`: VB-CABLE backend wrapper. +- `src/platform/windows/mic_write.cpp`: device discovery, WASAPI initialization, Opus decode, and VB-CABLE rendering. +- `src_assets/common/assets/web/Troubleshooting.vue`: Remote Microphone Debug UI. + +## Windows Requirements + +- Install VB-CABLE on the host. +- Ensure the playback endpoint `CABLE Input` exists and is enabled. +- In host applications, select `CABLE Output` as the microphone/recording source. +- Enable `stream_mic` in Vibepollo. +- Use a client build that supports microphone redirection. + +## Compatible Client Builds + +Microphone passthrough requires matching client support: + +- Desktop: [logabell/moonlight-qt-mic](https://github.com/logabell/moonlight-qt-mic) +- Android (Artemis): [logabell/moonlight-android](https://github.com/logabell/moonlight-android) +- Shared protocol library: [logabell/moonlight-common-c-mic](https://github.com/logabell/moonlight-common-c-mic) + +## Configuration Notes + +- `stream_mic` enables the host microphone redirect path. +- `mic_backend` defaults to `vb_cable` on Windows in this fork. +- On Windows, Vibepollo currently auto-detects the VB-CABLE render endpoint and standardizes on the VB-CABLE backend. +- `mic_device` is mainly relevant on non-Windows platforms. The Windows path currently targets VB-CABLE automatically. + +## Debugging + +The Troubleshooting page on Windows exposes a Remote Microphone Debug panel that shows: + +- whether the client is sending packets +- whether Vibepollo is decoding microphone frames +- whether Vibepollo is rendering into VB-CABLE +- whether non-silent input is being detected +- the most recent mic errors and recent mic events + +This view is intended to quickly separate client capture problems from host decode/render problems. diff --git a/src/audio.cpp b/src/audio.cpp index 716ed7bfa..d3a71278b 100644 --- a/src/audio.cpp +++ b/src/audio.cpp @@ -3,6 +3,10 @@ * @brief Definitions for audio capture and encoding. */ // standard includes +#include +#include +#include +#include #include // lib includes @@ -23,6 +27,52 @@ namespace audio { using opus_t = util::safe_ptr; using sample_queue_t = std::shared_ptr>>; + namespace { + struct mic_debug_state_t { + std::mutex mutex; + mic_debug_snapshot_t snapshot; + std::chrono::steady_clock::time_point last_packet_time {}; + std::chrono::steady_clock::time_point last_decode_time {}; + std::chrono::steady_clock::time_point last_render_time {}; + bool has_last_packet_time {false}; + bool has_last_decode_time {false}; + bool has_last_render_time {false}; + std::deque recent_events; + }; + + mic_debug_state_t &mic_debug_state() { + static mic_debug_state_t state; + return state; + } + + void append_mic_event(mic_debug_state_t &state, const std::string &message) { + const auto now = std::chrono::system_clock::now(); + const auto tt = std::chrono::system_clock::to_time_t(now); + std::tm tm {}; +#ifdef _WIN32 + localtime_s(&tm, &tt); +#else + localtime_r(&tt, &tm); +#endif + char timestamp[16] {}; + std::strftime(timestamp, sizeof(timestamp), "%H:%M:%S", &tm); + state.recent_events.push_front(std::string {timestamp} + " " + message); + while (state.recent_events.size() > 12) { + state.recent_events.pop_back(); + } + } + + void set_mic_state_locked(mic_debug_state_t &state, const std::string &status) { + state.snapshot.state = status; + append_mic_event(state, status); + } + + audio_ctx_ref_t &mic_redirect_audio_ctx() { + static audio_ctx_ref_t ref; + return ref; + } + } // namespace + static int start_audio_control(audio_ctx_t &ctx); static void stop_audio_control(audio_ctx_t &); static void apply_surround_params(opus_stream_config_t &stream, const stream_params_t ¶ms); @@ -287,6 +337,209 @@ namespace audio { return ctx.control->is_sink_available(sink); } + int init_mic_redirect_device() { + auto &held_ref = mic_redirect_audio_ctx(); + if (!held_ref) { + held_ref = get_audio_ctx_ref(); + } + + auto &ref = held_ref; + if (!ref || !ref->control) { + mic_debug_on_backend_error("Audio control is unavailable; microphone redirection could not initialize"); + return -1; + } + + return ref->control->init_mic_redirect_device(); + } + + void release_mic_redirect_device() { + auto &ref = mic_redirect_audio_ctx(); + if (!ref || !ref->control) { + ref = {}; + return; + } + + ref->control->release_mic_redirect_device(); + ref = {}; + } + + int write_mic_data(const char *data, std::size_t len, std::uint16_t sequence_number) { + auto &held_ref = mic_redirect_audio_ctx(); + auto ref = held_ref ? held_ref : get_audio_ctx_ref(); + if (!ref || !ref->control) { + BOOST_LOG(warning) << "Client microphone packet rejected before decode because audio control is unavailable" + << " [seq=" << sequence_number << ", len=" << len << ']'; + mic_debug_on_packet_dropped(sequence_number, "Audio control is unavailable while writing microphone data"); + return -1; + } + + return ref->control->write_mic_data(data, len, sequence_number); + } + + mic_debug_snapshot_t get_mic_debug_snapshot() { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + + auto snapshot = state.snapshot; + const auto now = std::chrono::steady_clock::now(); + if (state.has_last_packet_time) { + snapshot.last_packet_age_ms = std::chrono::duration_cast(now - state.last_packet_time).count(); + } + if (state.has_last_decode_time) { + snapshot.last_decode_age_ms = std::chrono::duration_cast(now - state.last_decode_time).count(); + } + if (state.has_last_render_time) { + snapshot.last_render_age_ms = std::chrono::duration_cast(now - state.last_render_time).count(); + } + snapshot.recent_events.assign(state.recent_events.begin(), state.recent_events.end()); + snapshot.signal_detected = snapshot.last_input_level >= 0.02 && snapshot.last_decode_age_ms >= 0 && snapshot.last_decode_age_ms < 3000; + return snapshot; + } + + void mic_debug_on_session_start(const std::string &client_name, bool encryption_enabled) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot = {}; + state.snapshot.session_active = true; + state.snapshot.mic_requested = true; + state.snapshot.encryption_enabled = encryption_enabled; + state.snapshot.client_name = client_name; + state.snapshot.state = "Microphone redirection negotiated; waiting for client audio"; + state.snapshot.last_packet_age_ms = -1; + state.snapshot.last_decode_age_ms = -1; + state.snapshot.last_render_age_ms = -1; + state.has_last_packet_time = false; + state.has_last_decode_time = false; + state.has_last_render_time = false; + state.recent_events.clear(); + append_mic_event(state, "Microphone redirection negotiated for client [" + client_name + "]"); + } + + void mic_debug_on_session_stop(const std::string &reason) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.session_active = false; + state.snapshot.decode_active = false; + state.snapshot.render_active = false; + state.snapshot.signal_detected = false; + state.snapshot.state = reason.empty() ? "No active remote microphone session" : reason; + append_mic_event(state, reason.empty() ? "Remote microphone session ended" : reason); + } + + void mic_debug_on_backend_initialized(const std::string &backend_name) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.backend_initialized = true; + state.snapshot.backend_name = backend_name; + state.snapshot.last_error.clear(); + append_mic_event(state, "Microphone backend ready: " + backend_name); + } + + void mic_debug_on_backend_target(const std::string &target_device_name, int channels, std::uint32_t sample_rate) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.target_device_name = target_device_name; + state.snapshot.state = "Rendering client microphone into " + target_device_name; + append_mic_event(state, "Using host render target [" + target_device_name + "] at " + std::to_string(channels) + "ch/" + std::to_string(sample_rate) + "Hz"); + } + + void mic_debug_on_backend_error(const std::string &message) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.last_error = message; + state.snapshot.render_active = false; + state.snapshot.state = message; + append_mic_event(state, message); + } + + void mic_debug_on_packet_received(std::uint16_t sequence_number, std::size_t payload_len) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.first_packet_received = true; + state.snapshot.packets_received++; + state.snapshot.last_sequence_number = sequence_number; + state.snapshot.last_payload_size = payload_len; + state.last_packet_time = std::chrono::steady_clock::now(); + state.has_last_packet_time = true; + if (state.snapshot.packets_received == 1) { + set_mic_state_locked(state, "Receiving microphone packets from Moonlight"); + } + } + + void mic_debug_on_packet_decrypt_error(std::uint16_t sequence_number, const std::string &message) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.decrypt_errors++; + state.snapshot.last_sequence_number = sequence_number; + state.snapshot.last_error = message; + state.snapshot.state = message; + append_mic_event(state, message); + } + + void mic_debug_on_packet_dropped(std::uint16_t sequence_number, const std::string &message) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.packets_dropped++; + state.snapshot.last_sequence_number = sequence_number; + state.snapshot.last_error = message; + state.snapshot.state = message; + append_mic_event(state, message); + } + + void mic_debug_on_packet_decoded(std::uint16_t sequence_number, double normalized_level, bool silent) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.decode_active = true; + state.snapshot.packets_decoded++; + state.snapshot.last_sequence_number = sequence_number; + state.snapshot.last_input_level = normalized_level; + state.snapshot.last_error.clear(); + if (silent) { + state.snapshot.silent_packets++; + } + state.last_decode_time = std::chrono::steady_clock::now(); + state.has_last_decode_time = true; + if (state.snapshot.packets_decoded == 1) { + set_mic_state_locked(state, "Apollo decoded microphone audio from Moonlight"); + } + } + + void mic_debug_on_packet_rendered(std::uint16_t sequence_number, double normalized_level, bool silent) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.packets_rendered++; + state.snapshot.last_sequence_number = sequence_number; + state.snapshot.last_render_level = normalized_level; + state.snapshot.render_active = true; + state.snapshot.last_error.clear(); + state.last_render_time = std::chrono::steady_clock::now(); + state.has_last_render_time = true; + if (state.snapshot.packets_rendered == 1) { + set_mic_state_locked(state, "Apollo is rendering microphone audio into VB-Cable"); + } + } + + void mic_debug_on_decode_error(std::uint16_t sequence_number, const std::string &message) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.decode_errors++; + state.snapshot.last_sequence_number = sequence_number; + state.snapshot.last_error = message; + state.snapshot.state = message; + append_mic_event(state, message); + } + + void mic_debug_on_render_error(std::uint16_t sequence_number, const std::string &message) { + auto &state = mic_debug_state(); + std::lock_guard lock(state.mutex); + state.snapshot.render_errors++; + state.snapshot.last_sequence_number = sequence_number; + state.snapshot.last_error = message; + state.snapshot.render_active = false; + state.snapshot.state = message; + append_mic_event(state, message); + } + int map_stream(int channels, bool quality) { int shift = quality ? 1 : 0; switch (channels) { diff --git a/src/audio.h b/src/audio.h index 1db35f17d..dff4b45c6 100644 --- a/src/audio.h +++ b/src/audio.h @@ -9,7 +9,11 @@ #include "thread_safe.h" #include "utility.h" +#include #include +#include +#include +#include namespace audio { enum stream_config_e : int { @@ -78,6 +82,38 @@ namespace audio { using packet_t = std::pair; using audio_ctx_ref_t = safe::shared_t::ptr_t; + struct mic_debug_snapshot_t { + bool session_active {}; + bool mic_requested {}; + bool encryption_enabled {}; + bool backend_initialized {}; + bool first_packet_received {}; + bool decode_active {}; + bool render_active {}; + bool signal_detected {}; + std::uint64_t packets_received {}; + std::uint64_t packets_decoded {}; + std::uint64_t packets_rendered {}; + std::uint64_t packets_dropped {}; + std::uint64_t decrypt_errors {}; + std::uint64_t decode_errors {}; + std::uint64_t render_errors {}; + std::uint64_t silent_packets {}; + std::uint16_t last_sequence_number {}; + std::size_t last_payload_size {}; + double last_input_level {}; + double last_render_level {}; + std::int64_t last_packet_age_ms {-1}; + std::int64_t last_decode_age_ms {-1}; + std::int64_t last_render_age_ms {-1}; + std::string client_name; + std::string backend_name; + std::string target_device_name; + std::string state; + std::string last_error; + std::vector recent_events; + }; + void capture(safe::mail_t mail, config_t config, void *channel_data); /** @@ -107,4 +143,20 @@ namespace audio { * @examples_end */ bool is_audio_ctx_sink_available(const audio_ctx_t &ctx); + int init_mic_redirect_device(); + void release_mic_redirect_device(); + int write_mic_data(const char *data, std::size_t len, std::uint16_t sequence_number); + mic_debug_snapshot_t get_mic_debug_snapshot(); + void mic_debug_on_session_start(const std::string &client_name, bool encryption_enabled); + void mic_debug_on_session_stop(const std::string &reason = {}); + void mic_debug_on_backend_initialized(const std::string &backend_name); + void mic_debug_on_backend_target(const std::string &target_device_name, int channels, std::uint32_t sample_rate); + void mic_debug_on_backend_error(const std::string &message); + void mic_debug_on_packet_received(std::uint16_t sequence_number, std::size_t payload_len); + void mic_debug_on_packet_decrypt_error(std::uint16_t sequence_number, const std::string &message); + void mic_debug_on_packet_dropped(std::uint16_t sequence_number, const std::string &message); + void mic_debug_on_packet_decoded(std::uint16_t sequence_number, double normalized_level, bool silent); + void mic_debug_on_packet_rendered(std::uint16_t sequence_number, double normalized_level, bool silent); + void mic_debug_on_decode_error(std::uint16_t sequence_number, const std::string &message); + void mic_debug_on_render_error(std::uint16_t sequence_number, const std::string &message); } // namespace audio diff --git a/src/config.cpp b/src/config.cpp index 108415a83..1e6772815 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -838,7 +838,10 @@ namespace config { audio_t audio { {}, // audio_sink {}, // virtual_sink + "vb_cable", // mic_backend + {}, // mic_device true, // stream audio + false, // stream microphone true, // install_steam_drivers true, // keep_sink_default true, // auto_capture @@ -1702,7 +1705,10 @@ namespace config { string_f(vars, "audio_sink", audio.sink); string_f(vars, "virtual_sink", audio.virtual_sink); + string_f(vars, "mic_backend", audio.mic_backend); + string_f(vars, "mic_device", audio.mic_device); bool_f(vars, "stream_audio", audio.stream); + bool_f(vars, "stream_mic", audio.stream_mic); bool_f(vars, "install_steam_audio_drivers", audio.install_steam_drivers); bool_f(vars, "keep_sink_default", audio.keep_default); bool_f(vars, "auto_capture_sink", audio.auto_capture); diff --git a/src/config.h b/src/config.h index a2bb82398..cc85cc6f8 100644 --- a/src/config.h +++ b/src/config.h @@ -189,7 +189,10 @@ namespace config { struct audio_t { std::string sink; std::string virtual_sink; + std::string mic_backend; + std::string mic_device; bool stream; + bool stream_mic; bool install_steam_drivers; bool keep_default; bool auto_capture; diff --git a/src/confighttp.cpp b/src/confighttp.cpp index 9f851d62a..90debd38a 100644 --- a/src/confighttp.cpp +++ b/src/confighttp.cpp @@ -38,6 +38,7 @@ #include // local includes +#include "audio.h" #include "config.h" #include "confighttp.h" #include "crypto.h" @@ -2436,6 +2437,53 @@ namespace confighttp { send_response(response, output_tree); } + /** + * @brief Get the active remote microphone debug status. + * @param response The HTTP response object. + * @param request The HTTP request object. + */ + void getAudioDebug(resp_https_t response, req_https_t request) { + if (!authenticate(response, request)) { + return; + } + + print_req(request); + + const auto snapshot = audio::get_mic_debug_snapshot(); + nlohmann::json output_tree; + output_tree["status"] = true; + output_tree["sessionActive"] = snapshot.session_active; + output_tree["micRequested"] = snapshot.mic_requested; + output_tree["encryptionEnabled"] = snapshot.encryption_enabled; + output_tree["backendInitialized"] = snapshot.backend_initialized; + output_tree["firstPacketReceived"] = snapshot.first_packet_received; + output_tree["decodeActive"] = snapshot.decode_active; + output_tree["renderActive"] = snapshot.render_active; + output_tree["signalDetected"] = snapshot.signal_detected; + output_tree["packetsReceived"] = snapshot.packets_received; + output_tree["packetsDecoded"] = snapshot.packets_decoded; + output_tree["packetsRendered"] = snapshot.packets_rendered; + output_tree["packetsDropped"] = snapshot.packets_dropped; + output_tree["decryptErrors"] = snapshot.decrypt_errors; + output_tree["decodeErrors"] = snapshot.decode_errors; + output_tree["renderErrors"] = snapshot.render_errors; + output_tree["silentPackets"] = snapshot.silent_packets; + output_tree["lastSequenceNumber"] = snapshot.last_sequence_number; + output_tree["lastPayloadSize"] = snapshot.last_payload_size; + output_tree["lastInputLevel"] = snapshot.last_input_level; + output_tree["lastRenderLevel"] = snapshot.last_render_level; + output_tree["lastPacketAgeMs"] = snapshot.last_packet_age_ms; + output_tree["lastDecodeAgeMs"] = snapshot.last_decode_age_ms; + output_tree["lastRenderAgeMs"] = snapshot.last_render_age_ms; + output_tree["clientName"] = snapshot.client_name; + output_tree["backendName"] = snapshot.backend_name; + output_tree["targetDeviceName"] = snapshot.target_device_name; + output_tree["state"] = snapshot.state; + output_tree["lastError"] = snapshot.last_error; + output_tree["recentEvents"] = snapshot.recent_events; + send_response(response, output_tree); + } + /** * @brief Save the configuration settings. * @param response The HTTP response object. @@ -4579,6 +4627,7 @@ namespace confighttp { register_api_route("^/api/config$", "PATCH", patchConfig); register_api_route("^/api/metadata$", "GET", getMetadata); register_api_route("^/api/configLocale$", "GET", getLocale); + register_api_route("^/api/audio-debug$", "GET", getAudioDebug); register_api_route("^/api/restart$", "POST", restart); register_api_route("^/api/quit$", "POST", quit); #if defined(_WIN32) diff --git a/src/crypto.cpp b/src/crypto.cpp index 43f71b80a..fdb856f99 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -146,6 +146,18 @@ namespace crypto { return 0; } + static int init_decrypt_cbc(cipher_ctx_t &ctx, aes_t *key, aes_t *iv, bool padding) { + ctx.reset(EVP_CIPHER_CTX_new()); + + if (EVP_DecryptInit_ex(ctx.get(), EVP_aes_128_cbc(), nullptr, key->data(), iv->data()) != 1) { + return -1; + } + + EVP_CIPHER_CTX_set_padding(ctx.get(), padding); + + return 0; + } + int gcm_t::decrypt(const std::string_view &tagged_cipher, std::vector &plaintext, aes_t *iv) { if (!decrypt_ctx && init_decrypt_gcm(decrypt_ctx, &key, iv, padding)) { return -1; @@ -305,6 +317,31 @@ namespace crypto { return update_outlen + final_outlen; } + int cbc_t::decrypt(const std::string_view &cipher, std::vector &plaintext, aes_t *iv) { + if (!decrypt_ctx && init_decrypt_cbc(decrypt_ctx, &key, iv, padding)) { + return -1; + } + + if (EVP_DecryptInit_ex(decrypt_ctx.get(), nullptr, nullptr, nullptr, iv->data()) != 1) { + return -1; + } + + plaintext.resize(round_to_pkcs7_padded(cipher.size())); + + int update_outlen, final_outlen; + + if (EVP_DecryptUpdate(decrypt_ctx.get(), plaintext.data(), &update_outlen, (const std::uint8_t *) cipher.data(), cipher.size()) != 1) { + return -1; + } + + if (EVP_DecryptFinal_ex(decrypt_ctx.get(), plaintext.data() + update_outlen, &final_outlen) != 1) { + return -1; + } + + plaintext.resize(update_outlen + final_outlen); + return 0; + } + ecb_t::ecb_t(const aes_t &key, bool padding): cipher_t {EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_new(), key, padding} { } diff --git a/src/crypto.h b/src/crypto.h index a0c30d61c..27960c784 100644 --- a/src/crypto.h +++ b/src/crypto.h @@ -237,6 +237,7 @@ namespace crypto { * @return The total length of the ciphertext written into cipher. Returns -1 in case of an error. */ int encrypt(const std::string_view &plaintext, std::uint8_t *cipher, aes_t *iv); + int decrypt(const std::string_view &cipher, std::vector &plaintext, aes_t *iv); }; } // namespace cipher } // namespace crypto diff --git a/src/platform/common.h b/src/platform/common.h index 1836ca76d..81c72ef2c 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -6,6 +6,7 @@ // standard includes #include +#include #include #include #include @@ -535,6 +536,10 @@ namespace platf { virtual std::unique_ptr microphone(const std::uint8_t *mapping, int channels, std::uint32_t sample_rate, std::uint32_t frame_size) = 0; + virtual int init_mic_redirect_device() = 0; + virtual void release_mic_redirect_device() = 0; + virtual int write_mic_data(const char *data, std::size_t len, std::uint16_t sequence_number) = 0; + /** * @brief Check if the audio sink is available in the system. * @param sink Sink to be checked. diff --git a/src/platform/linux/audio.cpp b/src/platform/linux/audio.cpp index 0e53e939b..86dbfdd40 100644 --- a/src/platform/linux/audio.cpp +++ b/src/platform/linux/audio.cpp @@ -3,12 +3,14 @@ * @brief Definitions for audio control on Linux. */ // standard includes +#include #include #include #include // lib includes #include +#include #include #include #include @@ -102,6 +104,71 @@ namespace platf { return mic; } + struct mic_redirect_t { + util::safe_ptr sink; + util::safe_ptr decoder; + + int init(const std::string &sink_name) { + int opus_error = OPUS_OK; + decoder.reset(opus_decoder_create(48000, 1, &opus_error)); + if (!decoder || opus_error != OPUS_OK) { + BOOST_LOG(error) << "Couldn't create Opus decoder for microphone redirection: "sv << opus_strerror(opus_error); + return -1; + } + + pa_sample_spec ss {PA_SAMPLE_S16NE, 48000, 1}; + pa_buffer_attr attr { + .maxlength = uint32_t(-1), + .tlength = uint32_t(960 * sizeof(opus_int16) * 6), + .prebuf = uint32_t(-1), + .minreq = uint32_t(-1), + .fragsize = uint32_t(-1), + }; + + int status = 0; + sink.reset(pa_simple_new(nullptr, "sunshine", PA_STREAM_PLAYBACK, sink_name.c_str(), "sunshine-mic", &ss, nullptr, &attr, &status)); + if (!sink) { + BOOST_LOG(error) << "Couldn't open PulseAudio sink for microphone redirection ["sv << sink_name << "]: "sv << pa_strerror(status); + decoder.reset(); + return -1; + } + + return 0; + } + + int write_data(const char *data, std::size_t len, std::uint16_t sequence_number) { + (void) sequence_number; + + if (!sink || !decoder || data == nullptr || len == 0) { + return -1; + } + + std::array pcm {}; + auto decoded = opus_decode(decoder.get(), reinterpret_cast(data), static_cast(len), pcm.data(), static_cast(pcm.size()), 0); + if (decoded <= 0) { + return -1; + } + + int status = 0; + if (pa_simple_write(sink.get(), pcm.data(), decoded * sizeof(opus_int16), &status) < 0) { + BOOST_LOG(debug) << "PulseAudio microphone write failed: "sv << pa_strerror(status); + return -1; + } + + return decoded; + } + + void cleanup() { + if (sink) { + int status = 0; + pa_simple_drain(sink.get(), &status); + } + + sink.reset(); + decoder.reset(); + } + }; + namespace pa { template struct add_const_helper; @@ -186,6 +253,7 @@ namespace platf { loop_t loop; ctx_t ctx; std::string requested_sink; + std::unique_ptr mic_redirect_device; struct { std::uint32_t stereo = PA_INVALID_INDEX; @@ -459,6 +527,42 @@ namespace platf { return ::platf::microphone(mapping, channels, sample_rate, frame_size, get_monitor_name(sink_name)); } + int init_mic_redirect_device() override { + if (mic_redirect_device) { + return 0; + } + + std::string sink_name = config::audio.mic_device; + if (sink_name.empty()) { + BOOST_LOG(warning) << "Set config option [stream_mic] with [mic_device] pointing to a virtual PulseAudio/PipeWire sink to enable microphone redirection"sv; + return -1; + } + + auto device = std::make_unique(); + if (device->init(sink_name) != 0) { + return -1; + } + + BOOST_LOG(info) << "Client microphone redirection target sink: "sv << sink_name; + mic_redirect_device = std::move(device); + return 0; + } + + void release_mic_redirect_device() override { + if (mic_redirect_device) { + mic_redirect_device->cleanup(); + mic_redirect_device.reset(); + } + } + + int write_mic_data(const char *data, std::size_t len, std::uint16_t sequence_number) override { + if (!mic_redirect_device) { + return -1; + } + + return mic_redirect_device->write_data(data, len, sequence_number); + } + bool is_sink_available(const std::string &sink) override { BOOST_LOG(warning) << "audio_control_t::is_sink_available() unimplemented: "sv << sink; return true; @@ -495,6 +599,7 @@ namespace platf { } ~server_t() override { + release_mic_redirect_device(); unload_null(index.stereo); unload_null(index.surround51); unload_null(index.surround71); diff --git a/src/platform/macos/microphone.mm b/src/platform/macos/microphone.mm index 06b9c19a8..169277653 100644 --- a/src/platform/macos/microphone.mm +++ b/src/platform/macos/microphone.mm @@ -78,6 +78,21 @@ int set_sink(const std::string &sink) override { return mic; } + int init_mic_redirect_device() override { + BOOST_LOG(warning) << "Client microphone redirection is not implemented on macOS yet"sv; + return -1; + } + + void release_mic_redirect_device() override { + } + + int write_mic_data(const char *data, std::size_t len, std::uint16_t sequence_number) override { + (void) data; + (void) len; + (void) sequence_number; + return -1; + } + bool is_sink_available(const std::string &sink) override { BOOST_LOG(warning) << "audio_control_t::is_sink_available() unimplemented: "sv << sink; return true; diff --git a/src/platform/windows/apollo_vmic.cpp b/src/platform/windows/apollo_vmic.cpp new file mode 100644 index 000000000..f6f4244e1 --- /dev/null +++ b/src/platform/windows/apollo_vmic.cpp @@ -0,0 +1,60 @@ +/** + * @file src/platform/windows/apollo_vmic.cpp + * @brief VB-CABLE backend for Windows host-side mic injection. + */ +#include "apollo_vmic.h" + +#include "mic_write.h" +#include "src/logging.h" + +namespace platf::audio { + apollo_vmic_t::~apollo_vmic_t() = default; + + std::string_view apollo_vmic_t::backend_id() const { + return "vb_cable"; + } + + bool apollo_vmic_t::log_missing_driver_once() { + if (missing_driver_logged) { + return false; + } + + missing_driver_logged = true; + BOOST_LOG(warning) + << "VB-CABLE microphone backend is unavailable. Install VB-CABLE and ensure the " + << "\"CABLE Input\" playback endpoint is present. Host applications should capture from " + << "\"CABLE Output\"."; + return true; + } + + int apollo_vmic_t::init() { + if (!speaker_backend) { + speaker_backend = std::make_unique( + "vb_cable", + std::vector { + L"CABLE Input", + L"VB-Audio Virtual Cable", + } + ); + } + + if (speaker_backend->init() == 0) { + return 0; + } + + speaker_backend.reset(); + log_missing_driver_once(); + return -1; + } + + int apollo_vmic_t::write_data(const char *data, std::size_t len, std::uint16_t sequence_number) { + if (!speaker_backend) { + BOOST_LOG(warning) << "Client microphone packet rejected before decode because the VB-CABLE speaker backend is missing" + << " [seq=" << sequence_number << ", len=" << len << ']'; + log_missing_driver_once(); + return -1; + } + + return speaker_backend->write_data(data, len, sequence_number); + } +} // namespace platf::audio diff --git a/src/platform/windows/apollo_vmic.h b/src/platform/windows/apollo_vmic.h new file mode 100644 index 000000000..e48e764d3 --- /dev/null +++ b/src/platform/windows/apollo_vmic.h @@ -0,0 +1,38 @@ +/** + * @file src/platform/windows/apollo_vmic.h + * @brief VB-CABLE microphone backend definitions. + */ +#pragma once + +#include +#include +#include +#include + +namespace platf::audio { + class mic_write_wasapi_t; + + class mic_redirect_backend_t { + public: + virtual ~mic_redirect_backend_t() = default; + + virtual std::string_view backend_id() const = 0; + virtual int init() = 0; + virtual int write_data(const char *data, std::size_t len, std::uint16_t sequence_number) = 0; + }; + + class apollo_vmic_t final: public mic_redirect_backend_t { + public: + ~apollo_vmic_t() override; + + std::string_view backend_id() const override; + int init() override; + int write_data(const char *data, std::size_t len, std::uint16_t sequence_number) override; + + private: + bool log_missing_driver_once(); + + bool missing_driver_logged = false; + std::unique_ptr speaker_backend; + }; +} // namespace platf::audio diff --git a/src/platform/windows/audio.cpp b/src/platform/windows/audio.cpp index c6db800c6..f4357dd99 100644 --- a/src/platform/windows/audio.cpp +++ b/src/platform/windows/audio.cpp @@ -9,6 +9,7 @@ #include #include #include +#include // platform includes #include @@ -20,6 +21,8 @@ #include // local includes +#include "apollo_vmic.h" +#include "mic_write.h" #include "misc.h" #include "src/config.h" #include "src/logging.h" @@ -202,6 +205,22 @@ namespace { return result; } + std::string normalize_mic_backend_name(const std::string &backend_name) { + if (backend_name.empty() || backend_name == "vb_cable") { + return "vb_cable"; + } + + if (backend_name == "auto" || backend_name == "legacy_device" || backend_name == "apollo_virtual_mic" || backend_name == "steam_streaming_microphone") { + BOOST_LOG(warning) << "Windows microphone backend ["sv << backend_name + << "] is no longer supported in Apollo Mic. Using VB-CABLE instead."; + return "vb_cable"; + } + + BOOST_LOG(warning) << "Unknown microphone backend ["sv << backend_name + << "], using VB-CABLE"; + return "vb_cable"; + } + } // namespace using namespace std::literals; @@ -894,6 +913,43 @@ namespace platf::audio { return mic; } + int init_mic_redirect_device() override { + if (mic_redirect_device) { + return 0; + } + + config::audio.mic_backend = normalize_mic_backend_name(config::audio.mic_backend); + + auto device = std::make_unique(); + if (device->init() == 0) { + active_mic_backend = std::string {device->backend_id()}; + BOOST_LOG(info) << "Client microphone redirection backend: " << active_mic_backend; + mic_redirect_device = std::move(device); + return 0; + } + + BOOST_LOG(warning) << "Client microphone redirection is unavailable because VB-CABLE is not installed or not accessible. " + << "Re-run the Apollo installer to let it install VB-CABLE automatically, or install it from www.vb-cable.com, " + << "then use \"CABLE Output\" as the host microphone in your applications."; + active_mic_backend.clear(); + return -1; + } + + void release_mic_redirect_device() override { + mic_redirect_device.reset(); + active_mic_backend.clear(); + } + + int write_mic_data(const char *data, std::size_t len, std::uint16_t sequence_number) override { + if (!mic_redirect_device) { + BOOST_LOG(warning) << "Client microphone packet rejected before decode because no Windows microphone redirect device is active" + << " [seq=" << sequence_number << ", len=" << len << ']'; + return -1; + } + + return mic_redirect_device->write_data(data, len, sequence_number); + } + /** * If the requested sink is a virtual sink, meaning no speakers attached to * the host, then we can seamlessly set the format to stereo and surround sound. @@ -1511,13 +1567,7 @@ namespace platf::audio { return reset_result_e::success; } - public: - - /** - * @brief Installs the Steam Streaming Speakers driver, if present. - * @return `true` if installation was successful. - */ - bool install_steam_audio_drivers() { + bool install_driver_from_local_steam_inf(const wchar_t *driver_path_template, std::wstring_view driver_name, bool restore_default_output_device) { #ifdef STEAM_DRIVER_SUBDIR // MinGW's libnewdev.a is missing DiInstallDriverW() even though the headers have it, // so we have to load it at runtime. It's Vista or later, so it will always be available. @@ -1536,22 +1586,23 @@ namespace platf::audio { return false; } - // Get the current default audio device (if present) - auto old_default_dev = default_device(device_enum); + audio::device_t old_default_dev; + if (restore_default_output_device) { + old_default_dev = default_device(device_enum); + } - // Install the Steam Streaming Speakers driver WCHAR driver_path[MAX_PATH] = {}; - ExpandEnvironmentStringsW(STEAM_AUDIO_DRIVER_PATH, driver_path, ARRAYSIZE(driver_path)); + ExpandEnvironmentStringsW(driver_path_template, driver_path, ARRAYSIZE(driver_path)); if (fn_DiInstallDriverW(nullptr, driver_path, 0, nullptr)) { - BOOST_LOG(info) << "Successfully installed Steam Streaming Speakers"sv; + BOOST_LOG(info) << "Successfully installed "sv << driver_name; // Wait for 5 seconds to allow the audio subsystem to reconfigure things before // modifying the default audio device or enumerating devices again. Sleep(5000); - // If there was a previous default device, restore that original device as the - // default output device just in case installing the new one changed it. - if (old_default_dev) { + if (restore_default_output_device && old_default_dev) { + // If there was a previous default device, restore that original device as the + // default output device just in case installing the new one changed it. audio::wstring_t old_default_id; old_default_dev->GetId(&old_default_id); @@ -1565,25 +1616,33 @@ namespace platf::audio { auto err = GetLastError(); switch (err) { case ERROR_ACCESS_DENIED: - BOOST_LOG(warning) << "Administrator privileges are required to install Steam Streaming Speakers"sv; + BOOST_LOG(warning) << "Administrator privileges are required to install "sv << driver_name; break; case ERROR_FILE_NOT_FOUND: case ERROR_PATH_NOT_FOUND: - BOOST_LOG(info) << "Steam audio drivers not found. This is expected if you don't have Steam installed."sv; + BOOST_LOG(info) << "Steam audio drivers not found locally. Install Steam on the host to use "sv << driver_name << '.'; break; default: - BOOST_LOG(warning) << "Failed to install Steam audio drivers: "sv << err; + BOOST_LOG(warning) << "Failed to install "sv << driver_name << ": "sv << err; break; } return false; } #else - BOOST_LOG(warning) << "Unable to install Steam Streaming Speakers on unknown architecture"sv; + BOOST_LOG(warning) << "Unable to install "sv << driver_name << " on unknown architecture"sv; return false; #endif } + /** + * @brief Installs the Steam Streaming Speakers driver, if present. + * @return `true` if installation was successful. + */ + bool install_steam_audio_drivers() { + return install_driver_from_local_steam_inf(STEAM_AUDIO_DRIVER_PATH, L"Steam Streaming Speakers", true); + } + int init() { auto status = CoCreateInstance( CLSID_CPolicyConfigClient, @@ -1620,6 +1679,8 @@ namespace platf::audio { policy_t policy; audio::device_enum_t device_enum; std::string assigned_sink; + std::string active_mic_backend; + std::unique_ptr mic_redirect_device; }; } // namespace platf::audio diff --git a/src/platform/windows/mic_write.cpp b/src/platform/windows/mic_write.cpp new file mode 100644 index 000000000..151dc2509 --- /dev/null +++ b/src/platform/windows/mic_write.cpp @@ -0,0 +1,439 @@ +/** + * @file src/platform/windows/mic_write.cpp + * @brief Windows microphone redirection writer. + */ +#include "mic_write.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "misc.h" +#include "src/audio.h" +#include "src/config.h" +#include "src/logging.h" + +namespace platf::audio { + namespace { + constexpr PROPERTYKEY PKEY_Device_DeviceDesc { + {0xa45c254e, 0xdf1c, 0x4efd, {0x80, 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0}}, + 2 + }; + constexpr PROPERTYKEY PKEY_Device_FriendlyName { + {0xa45c254e, 0xdf1c, 0x4efd, {0x80, 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0}}, + 14 + }; + constexpr PROPERTYKEY PKEY_DeviceInterface_FriendlyName { + {0x026e516e, 0xb814, 0x414b, {0x83, 0xcd, 0x85, 0x6d, 0x6f, 0xef, 0x48, 0x22}}, + 2 + }; + + template + void co_task_free(T *ptr) { + if (ptr) { + CoTaskMemFree(ptr); + } + } + + using device_t = util::safe_ptr>; + using collection_t = util::safe_ptr>; + using prop_t = util::safe_ptr>; + using wstring_t = util::safe_ptr>; + + class prop_var_t { + public: + prop_var_t() { + PropVariantInit(&value); + } + + ~prop_var_t() { + PropVariantClear(&value); + } + + PROPVARIANT value; + }; + + std::wstring get_prop_string(IPropertyStore *prop, REFPROPERTYKEY key) { + prop_var_t value; + if (FAILED(prop->GetValue(key, &value.value)) || value.value.vt != VT_LPWSTR || value.value.pwszVal == nullptr) { + return {}; + } + + return value.value.pwszVal; + } + + bool contains_case_insensitive(std::wstring haystack, std::wstring needle) { + std::transform(haystack.begin(), haystack.end(), haystack.begin(), ::towlower); + std::transform(needle.begin(), needle.end(), needle.begin(), ::towlower); + return haystack.find(needle) != std::wstring::npos; + } + + bool is_recoverable_device_error(HRESULT status) { + return status == AUDCLNT_E_DEVICE_INVALIDATED || + status == AUDCLNT_E_RESOURCES_INVALIDATED || + status == AUDCLNT_E_SERVICE_NOT_RUNNING; + } + + bool recover_device(mic_write_wasapi_t &writer, HRESULT status, const char *operation) { + if (!is_recoverable_device_error(status)) { + return false; + } + + BOOST_LOG(warning) << "Microphone playback device needs reinitialization after failure while " << operation + << ": 0x" << util::hex(status).to_string_view(); + + writer.cleanup(); + return writer.init() == 0; + } + } // namespace + + mic_write_wasapi_t::mic_write_wasapi_t(std::string backend_name, + std::vector autodetect_patterns, + std::string requested_device_name): + backend_name {std::move(backend_name)}, + requested_device_name {std::move(requested_device_name)}, + autodetect_patterns {std::move(autodetect_patterns)} { + } + + mic_write_wasapi_t::~mic_write_wasapi_t() { + cleanup(); + } + + std::string_view mic_write_wasapi_t::backend_id() const { + return backend_name; + } + + bool mic_write_wasapi_t::find_target_device(std::wstring &device_id, std::string &device_name) { + collection_t collection; + HRESULT status = device_enum->EnumAudioEndpoints(eRender, DEVICE_STATE_ACTIVE, &collection); + if (FAILED(status) || !collection) { + BOOST_LOG(error) << "Couldn't enumerate render devices for microphone redirection: 0x" << util::hex(status).to_string_view(); + return false; + } + + std::wstring requested_name = requested_device_name.empty() ? std::wstring {} : from_utf8(requested_device_name); + auto patterns = autodetect_patterns; + if (patterns.empty()) { + patterns = { + L"CABLE Input", + L"VB-Audio Virtual Cable", + }; + } + + UINT count = 0; + collection->GetCount(&count); + for (UINT index = 0; index < count; ++index) { + device_t device; + if (FAILED(collection->Item(index, &device)) || !device) { + continue; + } + + wstring_t id; + if (FAILED(device->GetId(&id)) || !id) { + continue; + } + + prop_t prop; + if (FAILED(device->OpenPropertyStore(STGM_READ, &prop)) || !prop) { + continue; + } + + auto friendly_name = get_prop_string(prop.get(), PKEY_Device_FriendlyName); + auto interface_name = get_prop_string(prop.get(), PKEY_DeviceInterface_FriendlyName); + auto description = get_prop_string(prop.get(), PKEY_Device_DeviceDesc); + + if (requested_name.empty() && + (contains_case_insensitive(friendly_name, L"16ch") || + contains_case_insensitive(interface_name, L"16ch") || + contains_case_insensitive(description, L"16ch"))) { + continue; + } + + bool matched = false; + if (!requested_name.empty()) { + matched = friendly_name == requested_name || interface_name == requested_name || description == requested_name || id.get() == requested_name; + } else { + for (const auto &pattern : patterns) { + if (contains_case_insensitive(friendly_name, pattern) || + contains_case_insensitive(interface_name, pattern) || + contains_case_insensitive(description, pattern)) { + matched = true; + break; + } + } + } + + if (!matched) { + continue; + } + + device_id = id.get(); + device_name = to_utf8(!friendly_name.empty() ? friendly_name : interface_name); + return true; + } + + return false; + } + + bool mic_write_wasapi_t::initialize_device() { + std::wstring device_id; + if (!find_target_device(device_id, target_device_name)) { + if (requested_device_name.empty()) { + BOOST_LOG(warning) << "No supported VB-CABLE playback device found. Install VB-CABLE and ensure \"CABLE Input\" is available."; + ::audio::mic_debug_on_backend_error("VB-CABLE was not found on the host. Install VB-CABLE and ensure CABLE Input is available."); + } else { + BOOST_LOG(warning) << "Requested microphone device not found: " << requested_device_name; + ::audio::mic_debug_on_backend_error("Requested microphone render device was not found: " + requested_device_name); + } + return false; + } + + device_t device; + HRESULT status = device_enum->GetDevice(device_id.c_str(), &device); + if (FAILED(status) || !device) { + BOOST_LOG(error) << "Couldn't open microphone playback device [" << target_device_name << "]: 0x" << util::hex(status).to_string_view(); + ::audio::mic_debug_on_backend_error("Could not open host microphone render device [" + target_device_name + "]"); + return false; + } + + status = device->Activate(IID_IAudioClient, CLSCTX_ALL, nullptr, (void **) &audio_client); + if (FAILED(status) || !audio_client) { + BOOST_LOG(error) << "Couldn't activate microphone playback client [" << target_device_name << "]: 0x" << util::hex(status).to_string_view(); + ::audio::mic_debug_on_backend_error("Could not activate the host microphone playback client for [" + target_device_name + "]"); + return false; + } + + std::array formats {{ + {WAVE_FORMAT_PCM, 1, 48000, 96000, 2, 16, 0}, + {WAVE_FORMAT_PCM, 2, 48000, 192000, 4, 16, 0}, + {WAVE_FORMAT_PCM, 1, 44100, 88200, 2, 16, 0}, + {WAVE_FORMAT_PCM, 2, 44100, 176400, 4, 16, 0}, + }}; + + HRESULT init_status = E_FAIL; + constexpr REFERENCE_TIME buffer_duration_100ns = 1000000; + for (const auto &format : formats) { + init_status = audio_client->Initialize( + AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, + buffer_duration_100ns, + 0, + &format, + nullptr + ); + if (SUCCEEDED(init_status)) { + active_format = format; + break; + } + } + + if (FAILED(init_status)) { + BOOST_LOG(error) << "Couldn't initialize microphone playback client [" << target_device_name << "]: 0x" << util::hex(init_status).to_string_view(); + ::audio::mic_debug_on_backend_error("Could not initialize the host microphone playback client for [" + target_device_name + "]"); + return false; + } + + status = audio_client->GetBufferSize(&buffer_frame_count); + if (FAILED(status)) { + BOOST_LOG(error) << "Couldn't query microphone playback buffer size: 0x" << util::hex(status).to_string_view(); + ::audio::mic_debug_on_backend_error("Could not query the microphone playback buffer size"); + return false; + } + + status = audio_client->GetService(IID_IAudioRenderClient, (void **) &audio_render); + if (FAILED(status) || !audio_render) { + BOOST_LOG(error) << "Couldn't acquire microphone playback render client: 0x" << util::hex(status).to_string_view(); + ::audio::mic_debug_on_backend_error("Could not acquire the microphone playback render client"); + return false; + } + + BOOST_LOG(info) << "Client microphone redirection target: " << target_device_name + << " (" << active_format.nChannels << "ch @" << active_format.nSamplesPerSec << "Hz)"; + ::audio::mic_debug_on_backend_target(target_device_name, active_format.nChannels, active_format.nSamplesPerSec); + + status = audio_client->Start(); + if (FAILED(status)) { + BOOST_LOG(error) << "Couldn't start microphone playback client: 0x" << util::hex(status).to_string_view(); + ::audio::mic_debug_on_backend_error("Could not start the microphone playback client"); + return false; + } + + return true; + } + + int mic_write_wasapi_t::init() { + int opus_error = OPUS_OK; + opus_decoder = opus_decoder_create(48000, 1, &opus_error); + if (opus_error != OPUS_OK || opus_decoder == nullptr) { + BOOST_LOG(error) << "Couldn't create Opus decoder for microphone redirection: " << opus_strerror(opus_error); + ::audio::mic_debug_on_backend_error("Could not create the Opus decoder for microphone redirection"); + return -1; + } + + HRESULT status = CoCreateInstance(CLSID_MMDeviceEnumerator, nullptr, CLSCTX_ALL, IID_IMMDeviceEnumerator, (void **) &device_enum); + if (FAILED(status) || !device_enum) { + BOOST_LOG(error) << "Couldn't create device enumerator for microphone redirection: 0x" << util::hex(status).to_string_view(); + ::audio::mic_debug_on_backend_error("Could not create the Windows audio device enumerator for microphone redirection"); + return -1; + } + + if (!initialize_device()) { + return -1; + } + + ::audio::mic_debug_on_backend_initialized(std::string {backend_name}); + + return 0; + } + + int mic_write_wasapi_t::write_data(const char *data, std::size_t len, std::uint16_t sequence_number) { + if (!audio_client || audio_render == nullptr || opus_decoder == nullptr || data == nullptr || len == 0) { + BOOST_LOG(warning) << "Client microphone packet rejected before decode because the WASAPI write path is not ready" + << " [seq=" << sequence_number + << ", len=" << len + << ", audio_client=" << static_cast(audio_client) + << ", audio_render=" << static_cast(audio_render != nullptr) + << ", opus_decoder=" << static_cast(opus_decoder != nullptr) + << ", data=" << static_cast(data != nullptr) << ']'; + return -1; + } + + std::array mono_pcm {}; + const auto decoded_frames = opus_decode(opus_decoder, reinterpret_cast(data), static_cast(len), mono_pcm.data(), static_cast(mono_pcm.size()), 0); + if (decoded_frames <= 0) { + BOOST_LOG(debug) << "Couldn't decode microphone Opus frame"; + ::audio::mic_debug_on_decode_error(sequence_number, "The host could not decode the incoming Opus microphone frame"); + return -1; + } + + int peak = 0; + for (int i = 0; i < decoded_frames; ++i) { + const int sample = mono_pcm[i] < 0 ? -mono_pcm[i] : mono_pcm[i]; + peak = std::max(peak, sample); + } + const double normalized_level = static_cast(peak) / 32767.0; + const bool silent = peak < 512; + ::audio::mic_debug_on_packet_decoded(sequence_number, normalized_level, silent); + + std::vector output_pcm; + output_pcm.reserve(decoded_frames * active_format.nChannels); + if (active_format.nChannels == 1) { + output_pcm.insert(output_pcm.end(), mono_pcm.begin(), mono_pcm.begin() + decoded_frames); + } else { + for (int i = 0; i < decoded_frames; ++i) { + output_pcm.push_back(mono_pcm[i]); + output_pcm.push_back(mono_pcm[i]); + } + } + + const auto bytes_per_frame = static_cast(active_format.nBlockAlign); + UINT32 frames_written = 0; + int empty_buffer_waits = 0; + while (frames_written < static_cast(decoded_frames)) { + UINT32 padding = 0; + auto status = audio_client->GetCurrentPadding(&padding); + if (FAILED(status)) { + if (recover_device(*this, status, "querying microphone playback padding")) { + return 0; + } + ::audio::mic_debug_on_render_error(sequence_number, "Could not query microphone playback padding"); + return -1; + } + + if (padding > buffer_frame_count) { + padding = 0; + } + + auto frames_available = buffer_frame_count - padding; + if (frames_available == 0) { + if (++empty_buffer_waits > 8) { + break; + } + Sleep(5); + continue; + } + + const auto frames_to_write = std::min(frames_available, static_cast(decoded_frames) - frames_written); + if (frames_to_write == 0) { + break; + } + + BYTE *buffer = nullptr; + status = audio_render->GetBuffer(frames_to_write, &buffer); + if (FAILED(status) || buffer == nullptr) { + if (FAILED(status) && recover_device(*this, status, "acquiring a microphone playback buffer")) { + return 0; + } + BOOST_LOG(debug) << "Couldn't acquire microphone playback buffer for [" << target_device_name << "]: 0x" + << util::hex(status).to_string_view(); + ::audio::mic_debug_on_render_error(sequence_number, "Could not acquire a microphone playback buffer from Windows"); + return -1; + } + + std::memcpy(buffer, output_pcm.data() + (frames_written * active_format.nChannels), frames_to_write * bytes_per_frame); + status = audio_render->ReleaseBuffer(frames_to_write, 0); + if (FAILED(status)) { + if (recover_device(*this, status, "releasing a microphone playback buffer")) { + return 0; + } + BOOST_LOG(debug) << "Couldn't release microphone playback buffer for [" << target_device_name << "]: 0x" + << util::hex(status).to_string_view(); + ::audio::mic_debug_on_render_error(sequence_number, "Could not release a microphone playback buffer to Windows"); + return -1; + } + + frames_written += frames_to_write; + empty_buffer_waits = 0; + } + + if (frames_written == 0) { + ::audio::mic_debug_on_render_error(sequence_number, "The microphone playback buffer stayed full long enough that this packet was skipped"); + return 0; + } + + if (frames_written < static_cast(decoded_frames)) { + BOOST_LOG(debug) << "Microphone playback buffer filled before the whole packet could be rendered for [" + << target_device_name << "], wrote " << frames_written << " of " << decoded_frames << " frames"; + } + + if (!first_packet_written_logged) { + first_packet_written_logged = true; + BOOST_LOG(info) << "Client microphone audio is being rendered into [" << target_device_name << ']'; + } + + ::audio::mic_debug_on_packet_rendered(sequence_number, normalized_level, silent); + + return decoded_frames; + } + + void mic_write_wasapi_t::cleanup() { + if (audio_client) { + audio_client->Stop(); + } + + if (audio_render != nullptr) { + audio_render->Release(); + audio_render = nullptr; + } + + audio_client.reset(); + device_enum.reset(); + + if (opus_decoder != nullptr) { + opus_decoder_destroy(opus_decoder); + opus_decoder = nullptr; + } + + buffer_frame_count = 0; + active_format = {}; + target_device_name.clear(); + first_packet_written_logged = false; + } +} // namespace platf::audio diff --git a/src/platform/windows/mic_write.h b/src/platform/windows/mic_write.h new file mode 100644 index 000000000..d22e58ac5 --- /dev/null +++ b/src/platform/windows/mic_write.h @@ -0,0 +1,56 @@ +/** + * @file src/platform/windows/mic_write.h + * @brief Windows microphone redirection writer. + */ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "apollo_vmic.h" +#include "src/platform/common.h" + +struct OpusDecoder; + +namespace platf::audio { + template + inline void release_com(T *ptr) { + if (ptr) { + ptr->Release(); + } + } + + class mic_write_wasapi_t: public mic_redirect_backend_t { + public: + mic_write_wasapi_t(std::string backend_name = "vb_cable", + std::vector autodetect_patterns = {}, + std::string requested_device_name = {}); + ~mic_write_wasapi_t(); + + std::string_view backend_id() const override; + int init() override; + int write_data(const char *data, std::size_t len, std::uint16_t sequence_number) override; + void cleanup(); + + private: + bool initialize_device(); + bool find_target_device(std::wstring &device_id, std::string &device_name); + + util::safe_ptr> device_enum; + util::safe_ptr> audio_client; + IAudioRenderClient *audio_render = nullptr; + OpusDecoder *opus_decoder = nullptr; + WAVEFORMATEX active_format {}; + UINT32 buffer_frame_count = 0; + std::string backend_name; + std::string requested_device_name; + std::vector autodetect_patterns; + std::string target_device_name; + bool first_packet_written_logged = false; + }; +} // namespace platf::audio diff --git a/src/rtsp.cpp b/src/rtsp.cpp index 867fd2de6..091a31a85 100644 --- a/src/rtsp.cpp +++ b/src/rtsp.cpp @@ -912,6 +912,11 @@ namespace rtsp_stream { uint32_t encryption_flags_supported = SS_ENC_CONTROL_V2 | SS_ENC_AUDIO; uint32_t encryption_flags_requested = SS_ENC_CONTROL_V2; + if (config::audio.stream_mic) { + encryption_flags_supported |= SS_ENC_MICROPHONE; + encryption_flags_requested |= SS_ENC_MICROPHONE; + } + // Determine the encryption desired for this remote endpoint auto encryption_mode = net::encryption_mode_for_address(sock.remote_endpoint().address()); if (encryption_mode != config::ENCRYPTION_MODE_NEVER) { @@ -947,6 +952,12 @@ namespace rtsp_stream { ss << "a=fmtp:97 surround-params="sv << session.surround_params << std::endl; } + if (config::audio.stream_mic) { + ss << "m=audio " << net::map_port(stream::MIC_STREAM_PORT) << " RTP/AVP 96" << std::endl; + ss << "a=rtpmap:96 opus/48000/1"sv << std::endl; + ss << "a=fmtp:96 minptime=10;useinbandfec=1"sv << std::endl; + } + for (int x = 0; x < audio::MAX_STREAM_CONFIG; ++x) { auto &stream_config = audio::stream_configs[x]; std::uint8_t mapping[platf::speaker::MAX_SPEAKERS]; @@ -1002,6 +1013,9 @@ namespace rtsp_stream { port = net::map_port(stream::VIDEO_STREAM_PORT); } else if (type == "control"sv) { port = net::map_port(stream::CONTROL_PORT); + } else if (type == "mic"sv && config::audio.stream_mic) { + port = net::map_port(stream::MIC_STREAM_PORT); + session.enable_mic = true; } else { cmd_not_found(sock, session, std::move(req)); @@ -1329,6 +1343,14 @@ namespace rtsp_stream { config.frame_generation_provider = session.frame_generation_provider; config.lossless_scaling_target_fps = session.lossless_scaling_target_fps; config.lossless_scaling_rtss_limit = session.lossless_scaling_rtss_limit; + + if (encryption_mode == config::ENCRYPTION_MODE_MANDATORY && + session.enable_mic && + !(config.encryptionFlagsEnabled & SS_ENC_MICROPHONE)) { + BOOST_LOG(warning) << "Disabling microphone redirection for ["sv << session.device_name << "] because the client did not negotiate microphone encryption"; + session.enable_mic = false; + } + auto stream_session = stream::session::alloc(config, session); server->insert(stream_session, session.client_uuid); diff --git a/src/rtsp.h b/src/rtsp.h index c310a0361..6d899b9a5 100644 --- a/src/rtsp.h +++ b/src/rtsp.h @@ -53,6 +53,7 @@ namespace rtsp_stream { bool input_only; bool host_audio; + bool enable_mic; int width; int height; int fps; diff --git a/src/stream.cpp b/src/stream.cpp index 4ec709800..d0613847d 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -148,7 +148,8 @@ namespace stream { enum class socket_e : int { video, ///< Video - audio ///< Audio + audio, ///< Audio + microphone ///< Microphone }; namespace session { @@ -333,6 +334,14 @@ namespace stream { AUDIO_FEC_HEADER fecHeader; }; + struct mic_packet_header_t { + std::uint8_t flags; + std::uint8_t packetType; + boost::endian::little_uint16_at sequenceNumber; + boost::endian::little_uint32_at timestamp; + boost::endian::little_uint32_at ssrc; + }; + #pragma pack(pop) constexpr std::size_t round_to_pkcs7_padded(std::size_t size) { @@ -340,7 +349,6 @@ namespace stream { } constexpr std::size_t MAX_AUDIO_PACKET_SIZE = 1400; - using audio_aes_t = std::array; using av_session_id_t = std::variant; // IP address or SS-Ping-Payload from RTSP handshake @@ -430,6 +438,7 @@ namespace stream { message_queue_queue_t message_queue_queue; std::thread recv_thread; + std::thread mic_thread; std::thread video_thread; std::thread audio_thread; std::thread control_thread; @@ -438,6 +447,7 @@ namespace stream { udp::socket video_sock {io_context}; udp::socket audio_sock {io_context}; + udp::socket mic_sock {io_context}; control_server_t control_server; }; @@ -490,6 +500,8 @@ namespace stream { audio_fec_packet_t fec_packet; std::unique_ptr qos; + bool enable_mic; + bool first_mic_packet_logged; } audio; struct { @@ -1748,6 +1760,99 @@ namespace stream { } } + session_t *find_mic_session(broadcast_ctx_t &ctx, const udp::endpoint &peer) { + auto lg = ctx.control_server._sessions.lock(); + for (auto *stream_session : *ctx.control_server._sessions) { + if (!stream_session->audio.enable_mic) { + continue; + } + + if (stream_session->state.load(std::memory_order_relaxed) != stream::session::state_e::RUNNING) { + continue; + } + + if (stream_session->audio.peer.address() == peer.address()) { + return stream_session; + } + } + + return nullptr; + } + + void micRecvThread(broadcast_ctx_t &ctx) { + auto broadcast_shutdown_event = mail::man->event(mail::broadcast_shutdown); + std::array buf {}; + udp::endpoint peer; + + while (!broadcast_shutdown_event->peek()) { + boost::system::error_code ec; + auto bytes = ctx.mic_sock.receive_from(asio::buffer(buf), peer, 0, ec); + + if (broadcast_shutdown_event->peek()) { + break; + } + + if (ec) { + if (ec == boost::asio::error::operation_aborted || + ec == boost::asio::error::bad_descriptor || + ec == boost::asio::error::connection_refused || + ec == boost::asio::error::connection_reset) { + continue; + } + + BOOST_LOG(debug) << "Couldn't receive microphone packet: "sv << ec.message(); + continue; + } + + if (bytes <= sizeof(mic_packet_header_t)) { + continue; + } + + auto *header = reinterpret_cast(buf.data()); + if (header->packetType != MIC_PACKET_TYPE_OPUS || header->ssrc != MIC_PACKET_MAGIC) { + continue; + } + + auto *session = find_mic_session(ctx, peer); + if (session == nullptr) { + continue; + } + + const auto sequence_number = static_cast(header->sequenceNumber); + const auto payload_len = bytes - sizeof(mic_packet_header_t); + const auto *payload = reinterpret_cast(buf.data() + sizeof(mic_packet_header_t)); + audio::mic_debug_on_packet_received(sequence_number, payload_len); + + if (!session->audio.first_mic_packet_logged) { + session->audio.first_mic_packet_logged = true; + BOOST_LOG(info) << "Received first client microphone packet for ["sv << session->device_name + << "] from ["sv << peer.address().to_string() << ':' << peer.port() + << "] with payload "sv << payload_len << " bytes"; + } + + std::vector decrypted_payload; + if (session->config.encryptionFlagsEnabled & SS_ENC_MICROPHONE) { + crypto::aes_t iv(16); + *(std::uint32_t *) iv.data() = util::endian::big(session->audio.avRiKeyId + sequence_number); + + if (session->audio.cipher.decrypt(std::string_view {reinterpret_cast(payload), payload_len}, decrypted_payload, &iv) != 0) { + BOOST_LOG(warning) << "Dropping encrypted microphone packet with invalid payload for ["sv << session->device_name + << "] sequence "sv << sequence_number; + audio::mic_debug_on_packet_decrypt_error(sequence_number, "Encrypted microphone packet could not be decrypted"); + continue; + } + + payload = decrypted_payload.data(); + } + + const auto decoded_payload_len = decrypted_payload.empty() ? payload_len : decrypted_payload.size(); + if (audio::write_mic_data(reinterpret_cast(payload), decoded_payload_len, sequence_number) < 0) { + BOOST_LOG(debug) << "Dropping microphone packet for ["sv << session->device_name << ']'; + audio::mic_debug_on_packet_dropped(sequence_number, "Host microphone render path rejected the packet"); + } + } + } + void videoBroadcastThread(udp::socket &sock) { auto shutdown_event = mail::man->event(mail::broadcast_shutdown); auto packets = mail::man->queue(mail::video_packets); @@ -2214,6 +2319,7 @@ namespace stream { auto control_port = net::map_port(CONTROL_PORT); auto video_port = net::map_port(VIDEO_STREAM_PORT); auto audio_port = net::map_port(AUDIO_STREAM_PORT); + auto mic_port = net::map_port(MIC_STREAM_PORT); if (ctx.control_server.bind(address_family, control_port)) { BOOST_LOG(error) << "Couldn't bind Control server to port ["sv << control_port << "], likely another process already bound to the port"sv; @@ -2264,6 +2370,20 @@ namespace stream { return -1; } + if (config::audio.stream_mic) { + ctx.mic_sock.open(protocol, ec); + if (ec) { + BOOST_LOG(fatal) << "Couldn't open socket for Microphone server: "sv << ec.message(); + return -1; + } + + ctx.mic_sock.bind(udp::endpoint(protocol, mic_port), ec); + if (ec) { + BOOST_LOG(fatal) << "Couldn't bind Microphone server to port ["sv << mic_port << "]: "sv << ec.message(); + return -1; + } + } + ctx.message_queue_queue = std::make_shared(30); // Restart the io_context in case it was stopped from a previous session. @@ -2275,6 +2395,9 @@ namespace stream { ctx.control_thread = std::thread {controlBroadcastThread, &ctx.control_server}; ctx.recv_thread = std::thread {recvThread, std::ref(ctx)}; + if (config::audio.stream_mic) { + ctx.mic_thread = std::thread {micRecvThread, std::ref(ctx)}; + } return 0; } @@ -2296,6 +2419,9 @@ namespace stream { ctx.video_sock.close(); ctx.audio_sock.close(); + if (ctx.mic_sock.is_open()) { + ctx.mic_sock.close(); + } video_packets.reset(); audio_packets.reset(); @@ -2308,6 +2434,10 @@ namespace stream { ctx.audio_thread.join(); BOOST_LOG(debug) << "Waiting for main control thread to end..."sv; ctx.control_thread.join(); + if (ctx.mic_thread.joinable()) { + BOOST_LOG(debug) << "Waiting for main microphone thread to end..."sv; + ctx.mic_thread.join(); + } BOOST_LOG(debug) << "All broadcasting threads ended"sv; broadcast_shutdown_event->reset(); @@ -2410,6 +2540,7 @@ namespace stream { namespace session { std::atomic_uint running_sessions; + std::atomic_uint running_mic_sessions; state_e state(session_t &session) { return session.state.load(std::memory_order_relaxed); @@ -2538,6 +2669,11 @@ namespace stream { exec_thread.detach(); } + if (session.audio.enable_mic && running_mic_sessions.fetch_sub(1, std::memory_order_acq_rel) == 1) { + audio::release_mic_redirect_device(); + audio::mic_debug_on_session_stop("Remote microphone session ended"); + } + // If this is the last session, invoke the platform callbacks const bool last_rtsp_session = --running_sessions == 0; if (last_rtsp_session) { @@ -2647,6 +2783,27 @@ namespace stream { session.audio.peer.address(addr); session.audio.peer.port(0); + if (session.audio.enable_mic) { + audio::mic_debug_on_session_start(session.device_name, (session.config.encryptionFlagsEnabled & SS_ENC_MICROPHONE) != 0); + if (running_mic_sessions.fetch_add(1, std::memory_order_acq_rel) == 0) { + if (audio::init_mic_redirect_device() != 0) { + running_mic_sessions.fetch_sub(1, std::memory_order_acq_rel); + session.audio.enable_mic = false; + audio::mic_debug_on_backend_error("Microphone backend could not initialize on the host"); + audio::mic_debug_on_session_stop("Microphone redirection requested, but the host backend could not initialize"); + BOOST_LOG(warning) << "Client microphone redirection is unavailable for ["sv << session.device_name << ']'; + } else { + BOOST_LOG(info) << "Client microphone redirection requested for ["sv << session.device_name + << "] with encryption "sv + << ((session.config.encryptionFlagsEnabled & SS_ENC_MICROPHONE) ? "enabled"sv : "disabled"sv); + } + } else { + BOOST_LOG(info) << "Client microphone redirection requested for ["sv << session.device_name + << "] with encryption "sv + << ((session.config.encryptionFlagsEnabled & SS_ENC_MICROPHONE) ? "enabled"sv : "disabled"sv); + } + } + session.pingTimeout = std::chrono::steady_clock::now() + config::stream.ping_timeout; session.audioThread = std::thread {audioThread, &session}; @@ -2870,6 +3027,8 @@ namespace stream { session->audio.avRiKeyId = util::endian::big(*(std::uint32_t *) launch_session.iv.data()); session->audio.sequenceNumber = 0; session->audio.timestamp = 0; + session->audio.enable_mic = launch_session.enable_mic && config::audio.stream_mic; + session->audio.first_mic_packet_logged = false; session->control.peer = nullptr; session->state.store(state_e::STOPPED, std::memory_order_relaxed); diff --git a/src/stream.h b/src/stream.h index 363c8d2fa..e721e5bd4 100644 --- a/src/stream.h +++ b/src/stream.h @@ -26,6 +26,7 @@ namespace stream { constexpr auto VIDEO_STREAM_PORT = 9; constexpr auto CONTROL_PORT = 10; constexpr auto AUDIO_STREAM_PORT = 11; + constexpr auto MIC_STREAM_PORT = 12; constexpr std::string_view video_format_name(int video_format) { switch (video_format) { diff --git a/src_assets/common/assets/web/Troubleshooting.vue b/src_assets/common/assets/web/Troubleshooting.vue index 90a2609d4..f83d1b575 100644 --- a/src_assets/common/assets/web/Troubleshooting.vue +++ b/src_assets/common/assets/web/Troubleshooting.vue @@ -96,6 +96,46 @@ + +
+
+

+ {{ translate('troubleshooting.remote_mic_title', 'Remote Microphone') }} +

+

+ {{ + translate( + 'troubleshooting.remote_mic_desc', + 'Use Moonlight microphone preview first, then follow the host-side validation stages below.', + ) + }} +

+
+ + + {{ micDebug?.state || translate('troubleshooting.remote_mic_idle', 'No active remote microphone session') }} + + +
+
+
+
{{ stage.label }}
+ + {{ micStageStateLabel(stage.state) }} + +
+

{{ stage.detail }}

+
+

+ {{ translate('troubleshooting.remote_mic_loading', 'Loading microphone status...') }} +

+
+
@@ -323,6 +363,217 @@ const closeAppPressed = ref(false); const closeAppStatus = ref(null as null | boolean); const restartPressed = ref(false); +type MicDebugSnapshot = { + sessionActive?: boolean; + firstPacketReceived?: boolean; + decodeActive?: boolean; + renderActive?: boolean; + signalDetected?: boolean; + packetsReceived?: number; + decodeErrors?: number; + renderErrors?: number; + lastPacketAgeMs?: number; + lastDecodeAgeMs?: number; + lastRenderAgeMs?: number; + targetDeviceName?: string; + state?: string; +}; + +const micDebug = ref(null); +let micDebugInterval: number | null = null; + +const isFreshMicAge = (ageMs?: number) => typeof ageMs === 'number' && ageMs >= 0 && ageMs < 3000; + +const micStages = computed(() => { + const debug = micDebug.value; + if (!debug) return []; + + const captureState = !debug.sessionActive ? 'idle' : debug.firstPacketReceived ? 'success' : 'warning'; + const captureDetail = !debug.sessionActive + ? translate( + 'troubleshooting.remote_mic_stage_capture_idle', + 'Start a remote session with microphone passthrough enabled.', + ) + : debug.firstPacketReceived + ? translate( + 'troubleshooting.remote_mic_stage_capture_ok', + 'Moonlight is sending microphone audio to Vibepollo. Confirm the local source with the Moonlight preview.', + ) + : translate( + 'troubleshooting.remote_mic_stage_capture_waiting', + 'Vibepollo negotiated microphone passthrough, but Moonlight has not sent microphone audio yet.', + ); + + let packetState: 'idle' | 'success' | 'warning' | 'danger' = 'idle'; + let packetDetail = translate( + 'troubleshooting.remote_mic_stage_packets_idle', + 'No active microphone session.', + ); + if (debug.sessionActive) { + if (!debug.firstPacketReceived) { + packetState = 'warning'; + packetDetail = translate( + 'troubleshooting.remote_mic_stage_packets_waiting', + 'Waiting for the first microphone packet from Moonlight.', + ); + } else if (isFreshMicAge(debug.lastPacketAgeMs)) { + packetState = 'success'; + packetDetail = translate( + 'troubleshooting.remote_mic_stage_packets_ok', + `Packets are arriving from Moonlight (${debug.lastPacketAgeMs} ms ago).`, + ); + } else { + packetState = 'warning'; + packetDetail = translate( + 'troubleshooting.remote_mic_stage_packets_stale', + 'Packets arrived earlier, but Vibepollo has not seen a fresh microphone packet recently.', + ); + } + } + + let decodeState: 'idle' | 'success' | 'warning' | 'danger' = 'idle'; + let decodeDetail = translate( + 'troubleshooting.remote_mic_stage_decode_idle', + 'No decoded microphone audio on the host yet.', + ); + if (debug.sessionActive) { + if ((debug.decodeErrors ?? 0) > 0 && !debug.decodeActive) { + decodeState = 'danger'; + decodeDetail = translate( + 'troubleshooting.remote_mic_stage_decode_error', + 'Vibepollo received microphone packets but could not decode them.', + ); + } else if (debug.decodeActive && isFreshMicAge(debug.lastDecodeAgeMs)) { + decodeState = 'success'; + decodeDetail = translate( + 'troubleshooting.remote_mic_stage_decode_ok', + `Vibepollo decoded microphone audio successfully (${debug.lastDecodeAgeMs} ms ago).`, + ); + } else if ((debug.packetsReceived ?? 0) > 0) { + decodeState = 'warning'; + decodeDetail = translate( + 'troubleshooting.remote_mic_stage_decode_waiting', + 'Vibepollo is receiving packets, but decoded microphone audio has not been confirmed yet.', + ); + } + } + + let renderState: 'idle' | 'success' | 'warning' | 'danger' = 'idle'; + let renderDetail = translate( + 'troubleshooting.remote_mic_stage_render_idle', + 'VB-CABLE rendering has not started.', + ); + if (debug.sessionActive) { + if ((debug.renderErrors ?? 0) > 0 && !debug.renderActive) { + renderState = 'danger'; + renderDetail = translate( + 'troubleshooting.remote_mic_stage_render_error', + 'Vibepollo decoded microphone audio, but writing it into VB-CABLE failed.', + ); + } else if (debug.renderActive && isFreshMicAge(debug.lastRenderAgeMs)) { + renderState = 'success'; + renderDetail = translate( + 'troubleshooting.remote_mic_stage_render_ok', + `Vibepollo is rendering microphone audio into ${debug.targetDeviceName || 'CABLE Input'} (${debug.lastRenderAgeMs} ms ago).`, + ); + } else if (debug.decodeActive) { + renderState = 'warning'; + renderDetail = translate( + 'troubleshooting.remote_mic_stage_render_waiting', + 'Vibepollo decoded microphone audio, but the VB-CABLE render stage has not completed yet.', + ); + } + } + + let signalState: 'idle' | 'success' | 'warning' | 'danger' = 'idle'; + let signalDetail = translate( + 'troubleshooting.remote_mic_stage_signal_idle', + 'No decoded microphone signal is available yet.', + ); + if (debug.sessionActive) { + if (debug.signalDetected) { + signalState = 'success'; + signalDetail = translate( + 'troubleshooting.remote_mic_stage_signal_ok', + 'Vibepollo is detecting non-silent microphone audio from Moonlight.', + ); + } else if (debug.decodeActive) { + signalState = 'warning'; + signalDetail = translate( + 'troubleshooting.remote_mic_stage_signal_silent', + 'Decoded microphone audio is currently silent or below the signal threshold.', + ); + } else if (debug.firstPacketReceived) { + signalState = 'warning'; + signalDetail = translate( + 'troubleshooting.remote_mic_stage_signal_waiting', + 'Packets are arriving, but Vibepollo has not decoded usable microphone audio yet.', + ); + } + } + + return [ + { + key: 'capture', + label: translate('troubleshooting.remote_mic_stage_capture', 'Moonlight capture/send'), + state: captureState, + detail: captureDetail, + }, + { + key: 'packets', + label: translate('troubleshooting.remote_mic_stage_packets', 'Packets arriving'), + state: packetState, + detail: packetDetail, + }, + { + key: 'decode', + label: translate('troubleshooting.remote_mic_stage_decode', 'Decoded on host'), + state: decodeState, + detail: decodeDetail, + }, + { + key: 'render', + label: translate('troubleshooting.remote_mic_stage_render', 'Rendered into VB-CABLE'), + state: renderState, + detail: renderDetail, + }, + { + key: 'signal', + label: translate('troubleshooting.remote_mic_stage_signal', 'Live signal detected'), + state: signalState, + detail: signalDetail, + }, + ]; +}); + +const micStatusType = computed(() => { + if (micDebug.value?.renderActive) return 'success'; + if (micDebug.value?.sessionActive) return 'warning'; + return 'default'; +}); + +function micStageClass(state: string) { + return ( + { + success: 'border-success/40 bg-success/10 text-success', + warning: 'border-warning/40 bg-warning/10 text-warning', + danger: 'border-error/40 bg-error/10 text-error', + idle: 'border-dark/10 dark:border-light/10 bg-surface/20', + }[state] || 'border-dark/10 dark:border-light/10 bg-surface/20' + ); +} + +function micStageStateLabel(state: string) { + return ( + { + success: translate('troubleshooting.remote_mic_state_ok', 'OK'), + warning: translate('troubleshooting.remote_mic_state_waiting', 'Waiting'), + danger: translate('troubleshooting.remote_mic_state_error', 'Error'), + idle: translate('troubleshooting.remote_mic_state_idle', 'Idle'), + }[state] || translate('troubleshooting.remote_mic_state_idle', 'Idle') + ); +} + const latestLogs = ref('Loading...'); const displayedLogs = ref('Loading...'); const logFilter = ref(''); @@ -952,6 +1203,18 @@ function restart() { http.post('./api/restart', {}, { validateStatus: () => true }); } +async function refreshMicDebug() { + if (platform.value !== 'windows') return; + try { + const response = await http.get('./api/audio-debug', { validateStatus: () => true }); + if (response.status >= 200 && response.status < 300) { + micDebug.value = response.data; + } + } catch (error) { + console.error('Error fetching microphone debug status:', error); + } +} + onMounted(async () => { loginDisposer = authStore.onLogin(() => { void refreshLogs(); @@ -968,10 +1231,16 @@ onMounted(async () => { logInterval = window.setInterval(refreshLogs, 5000); refreshLogs(); + + if (platform.value === 'windows') { + await refreshMicDebug(); + micDebugInterval = window.setInterval(refreshMicDebug, 1000); + } }); onBeforeUnmount(() => { if (logInterval) window.clearInterval(logInterval); + if (micDebugInterval) window.clearInterval(micDebugInterval); if (loginDisposer) loginDisposer(); if (searchDebounce !== null && typeof window !== 'undefined') { window.clearTimeout(searchDebounce); diff --git a/src_assets/common/assets/web/configs/tabs/AudioVideo.vue b/src_assets/common/assets/web/configs/tabs/AudioVideo.vue index 227c156c6..c997210ce 100644 --- a/src_assets/common/assets/web/configs/tabs/AudioVideo.vue +++ b/src_assets/common/assets/web/configs/tabs/AudioVideo.vue @@ -1,4 +1,4 @@ - - - + + + + +