diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c39c0f5bf..ed5cdb315 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -227,6 +227,8 @@ jobs: target: esp32s3 - path: 'components/ws-s3-touch/example' target: esp32s3 + - path: 'components/xiao-esp32s3-sense/example' + target: esp32s3 steps: - name: Checkout repo diff --git a/.github/workflows/upload_components.yml b/.github/workflows/upload_components.yml index 6472b731c..9c62f4b5c 100755 --- a/.github/workflows/upload_components.yml +++ b/.github/workflows/upload_components.yml @@ -135,6 +135,7 @@ jobs: components/ws-s3-geek components/ws-s3-lcd-1-47 components/ws-s3-touch + components/xiao-esp32s3-sense namespace: "espp" # use old version if this isn't a release for testing version: ${{ github.event.release && github.event.release.tag_name || 'v0.20.2' }} diff --git a/components/xiao-esp32s3-sense/CMakeLists.txt b/components/xiao-esp32s3-sense/CMakeLists.txt new file mode 100644 index 000000000..009de75d7 --- /dev/null +++ b/components/xiao-esp32s3-sense/CMakeLists.txt @@ -0,0 +1,6 @@ +idf_component_register( + INCLUDE_DIRS "include" + SRC_DIRS "src" + REQUIRES base_component driver esp_driver_gpio esp_driver_i2s fatfs led math task + REQUIRED_IDF_TARGETS "esp32s3" + ) diff --git a/components/xiao-esp32s3-sense/README.md b/components/xiao-esp32s3-sense/README.md new file mode 100644 index 000000000..09aa1248c --- /dev/null +++ b/components/xiao-esp32s3-sense/README.md @@ -0,0 +1,38 @@ +# XIAO ESP32S3 Sense Board Support Package (BSP) Component + +[![Badge](https://components.espressif.com/components/espp/xiao-esp32s3-sense/badge.svg)](https://components.espressif.com/components/espp/xiao-esp32s3-sense) + +The Seeed Studio XIAO ESP32S3 Sense combines the XIAO ESP32S3 module with the +Sense expansion board, which adds an OV2640 camera, a PDM microphone, and a +microSD card slot. + +The `espp::XiaoEsp32S3Sense` component provides a singleton hardware abstraction +for the board's documented camera, microphone, user-LED, and microSD pin +mappings. + +It also provides helper methods for creating default ESP-IDF PDM RX I2S +configuration structures for the onboard microphone, TimerCam-style user LED +control helpers including Gaussian breathing, and a T-Deck / T-Dongle-S3 style +microSD mount helper. + +Published helpers include: + +- `camera_pins()` +- `microphone_pins()` +- `sd_card_pins()` +- `initialize_sdcard()` +- `sdcard()` +- `user_led_pin()` +- `initialize_led()` +- `start_led_breathing()` +- `set_led_breathing_period()` +- `set_led_brightness()` +- `gaussian()` + +## Example + +The [example](./example) shows how to use the `espp::XiaoEsp32S3Sense` +component in a streamer-style application that serves OV2640 MJPEG video and +PDM-microphone PCM audio over RTSP while driving the onboard user LED with the +same breathing behavior used by the ESP32 Timer-Cam example, and performing a +simple microSD mount/write/read smoke test when a card is present. diff --git a/components/xiao-esp32s3-sense/example/CMakeLists.txt b/components/xiao-esp32s3-sense/example/CMakeLists.txt new file mode 100644 index 000000000..8a80fe562 --- /dev/null +++ b/components/xiao-esp32s3-sense/example/CMakeLists.txt @@ -0,0 +1,30 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.20) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +# add the component directories that we want to use +set(EXTRA_COMPONENT_DIRS + "../../../components/base_component" + "../../../components/cli" + "../../../components/format" + "../../../components/logger" + "../../../components/monitor" + "../../../components/rtsp" + "../../../components/socket" + "../../../components/task" + "../../../components/wifi" + "../../../components/xiao-esp32s3-sense" +) + +set( + COMPONENTS + "main esptool_py cli esp32-camera mdns monitor rtsp task wifi xiao-esp32s3-sense" + CACHE STRING + "List of components to include" + ) + +project(xiao_esp32s3_sense_example) + +set(CMAKE_CXX_STANDARD 20) diff --git a/components/xiao-esp32s3-sense/example/README.md b/components/xiao-esp32s3-sense/example/README.md new file mode 100644 index 000000000..f31a0601d --- /dev/null +++ b/components/xiao-esp32s3-sense/example/README.md @@ -0,0 +1,62 @@ +# XIAO ESP32S3 Sense Example + +This example turns the Seeed Studio XIAO ESP32S3 Sense into a small RTSP +camera streamer. + +It mirrors the newer ESP32-TimerCam streamer-style example: + +- connects to Wi-Fi +- breathes the onboard user LED quickly while disconnected, then more slowly once connected +- starts an RTSP server and advertises it over mDNS +- captures MJPEG video from the onboard OV2640 camera +- captures 16 kHz mono PCM audio from the onboard PDM microphone +- mounts the onboard microSD card and performs a simple file write/read smoke test when present +- serves both tracks from a single RTSP session +- exposes a simple serial CLI with Wi-Fi and memory-monitor commands + +## How to use example + +### Hardware Required + +This example is designed to run on the Seeed Studio XIAO ESP32S3 Sense. + +### Configuration + +Set the Wi-Fi credentials and RTSP port in `menuconfig`: + +```bash +idf.py menuconfig +``` + +Open **Camera Streamer Configuration** and set: + +- **WiFi SSID** +- **WiFi Password** +- **RTSP Server Port** +- **RTSP accept/session/control task stack sizes** as needed for your target + +The defaults are tuned for the XIAO ESP32S3 Sense, including a larger RTSP +control-task stack to avoid stack overflows in tasks named like +`RtspSession `. + +### Build and Flash + +Build the project and flash it to the board, then run monitor tool to view +serial output: + +```bash +idf.py -p PORT flash monitor +``` + +(Replace PORT with the name of the serial port to use.) + +(To exit the serial monitor, type `Ctrl-]`.) + +After the board joins Wi-Fi it advertises an `_rtsp._tcp` mDNS service and +starts streaming at: + +```text +rtsp://:/mjpeg/1 +``` + +Track 0 is MJPEG video and track 1 is L16/16000 mono audio. diff --git a/components/xiao-esp32s3-sense/example/main/CMakeLists.txt b/components/xiao-esp32s3-sense/example/main/CMakeLists.txt new file mode 100644 index 000000000..8df34d7bf --- /dev/null +++ b/components/xiao-esp32s3-sense/example/main/CMakeLists.txt @@ -0,0 +1,4 @@ +idf_component_register(SRC_DIRS "." + INCLUDE_DIRS "." + REQUIRES cli esp32-camera esp_driver_i2s mdns monitor rtsp task wifi + xiao-esp32s3-sense) diff --git a/components/xiao-esp32s3-sense/example/main/Kconfig.projbuild b/components/xiao-esp32s3-sense/example/main/Kconfig.projbuild new file mode 100644 index 000000000..00cd6bf88 --- /dev/null +++ b/components/xiao-esp32s3-sense/example/main/Kconfig.projbuild @@ -0,0 +1,51 @@ +menu "Camera Streamer Configuration" + + config RTSP_SERVER_PORT + int "RTSP Server Port" + default 8554 + help + The port number of the RTSP server. + + config RTSP_ACCEPT_TASK_STACK_SIZE + int "RTSP accept task stack size (bytes)" + range 2048 32768 + default 4096 + help + Stack size for the RTSP server accept task. + + config RTSP_SESSION_TASK_STACK_SIZE + int "RTSP session task stack size (bytes)" + range 2048 32768 + default 4096 + help + Stack size for the RTSP server session-dispatch task. + + config RTSP_CONTROL_TASK_STACK_SIZE + int "RTSP control task stack size (bytes)" + range 2048 32768 + default 8192 + help + Stack size for each per-client RTSP control task. Increase this if + the board reports a stack overflow in a task named like + "RtspSession ". + + config ESP_WIFI_SSID + string "WiFi SSID" + default "" + help + SSID (network name) for the camera streamer to connect to. + + config ESP_WIFI_PASSWORD + string "WiFi Password" + default "" + help + WiFi password (WPA or WPA2) for the camera streamer to use. + + config ESP_MAXIMUM_RETRY + int "Maximum retry" + default 5 + help + Set the Maximum retry to avoid station reconnecting to the AP unlimited when the AP is + really inexistent. + +endmenu diff --git a/components/xiao-esp32s3-sense/example/main/idf_component.yml b/components/xiao-esp32s3-sense/example/main/idf_component.yml new file mode 100644 index 000000000..5758325c3 --- /dev/null +++ b/components/xiao-esp32s3-sense/example/main/idf_component.yml @@ -0,0 +1,24 @@ +## IDF Component Manager Manifest File +dependencies: + idf: + version: '>=5.0' + espressif/mdns: '>=1.8' + espressif/esp32-camera: '>=2.0' + espp/cli: + version: '*' + override_path: ../../../cli + espp/monitor: + version: '*' + override_path: ../../../monitor + espp/rtsp: + version: '*' + override_path: ../../../rtsp + espp/task: + version: '*' + override_path: ../../../task + espp/wifi: + version: '*' + override_path: ../../../wifi + espp/xiao-esp32s3-sense: + version: '*' + override_path: ../.. diff --git a/components/xiao-esp32s3-sense/example/main/xiao_esp32s3_sense_example.cpp b/components/xiao-esp32s3-sense/example/main/xiao_esp32s3_sense_example.cpp new file mode 100644 index 000000000..455c746e0 --- /dev/null +++ b/components/xiao-esp32s3-sense/example/main/xiao_esp32s3_sense_example.cpp @@ -0,0 +1,613 @@ +#include "sdkconfig.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "cli.hpp" +#include "generic_packetizer.hpp" +#include "heap_monitor.hpp" +#include "rtsp_server.hpp" +#include "task.hpp" +#include "task_monitor.hpp" +#include "wifi_sta.hpp" +#include "wifi_sta_menu.hpp" +#include "xiao-esp32s3-sense.hpp" + +using namespace std::chrono_literals; + +static espp::Logger logger({.tag = "XIAO Camera Streamer", .level = espp::Logger::Verbosity::INFO}); + +namespace { +constexpr auto idle_capture_poll_period = 250ms; +constexpr auto dma_pressure_backoff = 250ms; +constexpr auto camera_capture_error_backoff = 100ms; +constexpr auto target_video_period = 100ms; +constexpr auto audio_capture_timeout = 50ms; +constexpr size_t min_dma_free_bytes_for_streaming = 12 * 1024; +constexpr size_t min_dma_largest_block_for_streaming = 4 * 1024; + +constexpr int video_track_id = 0; +constexpr int audio_track_id = 1; +constexpr uint32_t microphone_sample_rate_hz = + espp::XiaoEsp32S3Sense::microphone_default_sample_rate_hz(); +constexpr size_t microphone_frame_samples = microphone_sample_rate_hz / 50; // 20 ms +constexpr size_t microphone_frame_bytes = microphone_frame_samples * sizeof(int16_t); +constexpr char rtsp_path[] = "mjpeg/1"; + +template +bool wait_until_or_stop(std::mutex &m, std::condition_variable &cv, bool &task_notified, + const std::chrono::time_point &deadline) { + std::unique_lock lk(m); + auto stop_requested = cv.wait_until(lk, deadline, [&task_notified] { return task_notified; }); + task_notified = false; + return stop_requested; +} +} // namespace + +std::recursive_mutex server_mutex; +std::unique_ptr camera_task; +std::unique_ptr audio_task; +std::unique_ptr memory_monitor_task; +std::unique_ptr task_monitor; +std::shared_ptr rtsp_server; +i2s_chan_handle_t microphone_rx_channel{nullptr}; +bool mdns_started{false}; +std::atomic video_frames_streamed{0}; +std::atomic audio_frames_streamed{0}; + +esp_err_t initialize_camera(void); +esp_err_t initialize_microphone(void); +void deinitialize_microphone(void); +bool start_rtsp_server(std::string_view server_address, int server_port); +void stop_streaming(void); +bool initialize_and_test_sdcard(void); +bool camera_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); +bool audio_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); +bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); + +extern "C" void app_main(void) { + logger.info("Bootup"); + + static constexpr float disconnected_led_breathing_period = 1.0f; + static constexpr float connected_led_breathing_period = 3.5f; + + auto &sense = espp::XiaoEsp32S3Sense::get(); + auto microphone = sense.microphone_pins(); + auto sd_card = sense.sd_card_pins(); + logger.info("Using XIAO ESP32S3 Sense camera + microphone"); + logger.info("User LED pin: {}", static_cast(sense.user_led_pin())); + logger.info("Microphone pins: clk={}, data={}", static_cast(microphone.clk), + static_cast(microphone.data)); + logger.info("microSD pins: clk={}, mosi={}, miso={}, cs={}", static_cast(sd_card.clk), + static_cast(sd_card.mosi), static_cast(sd_card.miso), + static_cast(sd_card.cs)); + + if (!sense.initialize_led(disconnected_led_breathing_period)) { + logger.error("Could not initialize user LED"); + return; + } + sense.start_led_breathing(); + + initialize_and_test_sdcard(); + + logger.info("Initializing camera"); + auto err = initialize_camera(); + if (err != ESP_OK) { + logger.error("Could not initialize camera: {} '{}'", err, esp_err_to_name(err)); + return; + } + + logger.info("Initializing microphone"); + err = initialize_microphone(); + if (err != ESP_OK) { + logger.error("Could not initialize microphone: {} '{}'", err, esp_err_to_name(err)); + deinitialize_microphone(); + return; + } + + logger.info("Starting memory monitors"); + task_monitor = std::make_unique(espp::TaskMonitor::Config{.period = 30s}); + memory_monitor_task = espp::Task::make_unique(espp::Task::Config{ + .callback = memory_monitor_task_fn, + .task_config = + { + .name = "Memory Monitor", + .stack_size_bytes = 6 * 1024, + }, + .log_level = espp::Logger::Verbosity::WARN, + }); + + logger.info("Initializing WiFi"); + espp::WifiSta wifi_sta( + {.ssid = CONFIG_ESP_WIFI_SSID, + .password = CONFIG_ESP_WIFI_PASSWORD, + .num_connect_retries = CONFIG_ESP_MAXIMUM_RETRY, + .on_connected = + []() { + auto &sense = espp::XiaoEsp32S3Sense::get(); + logger.info("Connected to WiFi, waiting for IP address"); + sense.set_led_breathing_period(connected_led_breathing_period); + memory_monitor_task->start(); + }, + .on_disconnected = + []() { + auto &sense = espp::XiaoEsp32S3Sense::get(); + logger.info("WiFi disconnected, stopping RTSP streaming"); + sense.set_led_breathing_period(disconnected_led_breathing_period); + stop_streaming(); + memory_monitor_task->stop(); + }, + .on_got_ip = + [](ip_event_got_ip_t *eventdata) { + auto server_address = fmt::format("{}.{}.{}.{}", IP2STR(&eventdata->ip_info.ip)); + logger.info("got IP: {}", server_address); + stop_streaming(); + + if (esp_wifi_set_ps(WIFI_PS_NONE) != ESP_OK) { + logger.warn( + "Could not disable WiFi power save; RTSP streaming may hit TX backpressure"); + } else { + logger.info("Disabled WiFi power save for RTSP streaming"); + } + + if (!start_rtsp_server(server_address, CONFIG_RTSP_SERVER_PORT)) { + logger.error("RTSP server failed to start, not starting capture tasks"); + return; + } + + auto new_camera_task = espp::Task::make_unique(espp::Task::Config{ + .callback = camera_task_fn, + .task_config = {.name = "Camera Task", .priority = 10}, + .log_level = espp::Logger::Verbosity::WARN, + }); + if (!new_camera_task->start()) { + logger.error("Could not start camera task"); + stop_streaming(); + return; + } + + auto new_audio_task = espp::Task::make_unique(espp::Task::Config{ + .callback = audio_task_fn, + .task_config = + { + .name = "Audio Task", + .stack_size_bytes = 6 * 1024, + .priority = 9, + }, + .log_level = espp::Logger::Verbosity::WARN, + }); + if (!new_audio_task->start()) { + logger.error("Could not start audio task"); + new_camera_task.reset(); + stop_streaming(); + return; + } + + std::lock_guard lock(server_mutex); + camera_task = std::move(new_camera_task); + audio_task = std::move(new_audio_task); + }}); + + espp::WifiStaMenu sta_menu(wifi_sta); + auto root_menu = sta_menu.get(); + root_menu->Insert( + "memory", + [](std::ostream &out) { + out << "Video frames streamed: " << video_frames_streamed.load() << '\n'; + out << "Audio frames streamed: " << audio_frames_streamed.load() << '\n'; + out << espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA}) + << std::endl; + }, + "Display current heap monitor information."); + + cli::Cli cli(std::move(root_menu)); + cli::SetColor(); + cli.ExitAction([](auto &out) { out << "Goodbye and thanks for all the fish.\n"; }); + + espp::Cli input(cli); + input.SetInputHistorySize(10); + input.Start(); +} + +esp_err_t initialize_camera(void) { + auto &sense = espp::XiaoEsp32S3Sense::get(); + auto camera = sense.camera_pins(); + + static camera_config_t camera_config = { + .pin_pwdn = static_cast(camera.pwdn), + .pin_reset = static_cast(camera.reset), + .pin_xclk = static_cast(camera.xclk), + .pin_sccb_sda = static_cast(camera.sccb_sda), + .pin_sccb_scl = static_cast(camera.sccb_scl), + + .pin_d7 = static_cast(camera.d7), + .pin_d6 = static_cast(camera.d6), + .pin_d5 = static_cast(camera.d5), + .pin_d4 = static_cast(camera.d4), + .pin_d3 = static_cast(camera.d3), + .pin_d2 = static_cast(camera.d2), + .pin_d1 = static_cast(camera.d1), + .pin_d0 = static_cast(camera.d0), + .pin_vsync = static_cast(camera.vsync), + .pin_href = static_cast(camera.href), + .pin_pclk = static_cast(camera.pclk), + + .xclk_freq_hz = sense.camera_xclk_freq_hz(), + .ledc_timer = LEDC_TIMER_0, + .ledc_channel = LEDC_CHANNEL_0, + + .pixel_format = PIXFORMAT_JPEG, + .frame_size = FRAMESIZE_QVGA, + .jpeg_quality = 15, + .fb_count = 2, + .fb_location = CAMERA_FB_IN_PSRAM, + .grab_mode = CAMERA_GRAB_LATEST, + .sccb_i2c_port = I2C_NUM_0, + }; + + auto err = esp_camera_init(&camera_config); + if (err != ESP_OK) { + return err; + } + + if (auto *sensor = esp_camera_sensor_get(); sensor != nullptr) { + sensor->set_vflip(sensor, false); + sensor->set_hmirror(sensor, false); + } + return ESP_OK; +} + +esp_err_t initialize_microphone(void) { + if (microphone_rx_channel != nullptr) { + return ESP_OK; + } + + auto &sense = espp::XiaoEsp32S3Sense::get(); + i2s_chan_config_t channel_config = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_AUTO, I2S_ROLE_MASTER); + channel_config.auto_clear = true; + channel_config.dma_desc_num = 4; + channel_config.dma_frame_num = microphone_frame_samples; + + auto err = i2s_new_channel(&channel_config, nullptr, µphone_rx_channel); + if (err != ESP_OK) { + logger.error("Could not allocate I2S RX channel: {} '{}'", err, esp_err_to_name(err)); + microphone_rx_channel = nullptr; + return err; + } + + auto microphone_config = sense.microphone_config(microphone_sample_rate_hz); + err = i2s_channel_init_pdm_rx_mode(microphone_rx_channel, µphone_config); + if (err != ESP_OK) { + logger.error("Could not initialize PDM RX mode: {} '{}'", err, esp_err_to_name(err)); + deinitialize_microphone(); + return err; + } + + err = i2s_channel_enable(microphone_rx_channel); + if (err != ESP_OK) { + logger.error("Could not enable microphone channel: {} '{}'", err, esp_err_to_name(err)); + deinitialize_microphone(); + return err; + } + + logger.info("Microphone initialized for {} Hz mono PCM", microphone_sample_rate_hz); + return ESP_OK; +} + +void deinitialize_microphone(void) { + if (microphone_rx_channel == nullptr) { + return; + } + i2s_channel_disable(microphone_rx_channel); + i2s_del_channel(microphone_rx_channel); + microphone_rx_channel = nullptr; +} + +bool initialize_and_test_sdcard(void) { + auto &sense = espp::XiaoEsp32S3Sense::get(); + if (!sense.initialize_sdcard()) { + logger.warn("Could not initialize microSD card, continuing without storage"); + return false; + } + + constexpr char test_file_path[] = "/sdcard/xiao_sdcard_smoke_test.txt"; + constexpr std::string_view test_payload = "xiao-esp32s3-sense sdcard smoke test\n"; + + auto *file = std::fopen(test_file_path, "wb"); + if (file == nullptr) { + logger.error("Could not open '{}' for writing", test_file_path); + return false; + } + + size_t bytes_written = std::fwrite(test_payload.data(), 1, test_payload.size(), file); + std::fclose(file); + if (bytes_written != test_payload.size()) { + logger.error("microSD smoke test short write: wrote {} of {} bytes", bytes_written, + test_payload.size()); + std::remove(test_file_path); + return false; + } + + std::array readback{}; + file = std::fopen(test_file_path, "rb"); + if (file == nullptr) { + logger.error("Could not reopen '{}' for reading", test_file_path); + std::remove(test_file_path); + return false; + } + + size_t bytes_read = std::fread(readback.data(), 1, readback.size() - 1, file); + std::fclose(file); + std::remove(test_file_path); + + std::string_view readback_view(readback.data(), bytes_read); + if (readback_view != test_payload) { + logger.error("microSD smoke test readback mismatch: expected '{}', got '{}'", test_payload, + readback_view); + return false; + } + + logger.info("microSD smoke test passed using '{}'", test_file_path); + return true; +} + +bool start_rtsp_server(std::string_view server_address, int server_port) { + logger.info("Creating RTSP server at {}:{}", server_address, server_port); + + auto server = std::make_shared(espp::RtspServer::Config{ + .server_address = std::string(server_address), + .port = server_port, + .path = rtsp_path, + .log_level = espp::Logger::Verbosity::WARN, + .accept_task_stack_size_bytes = CONFIG_RTSP_ACCEPT_TASK_STACK_SIZE, + .session_task_stack_size_bytes = CONFIG_RTSP_SESSION_TASK_STACK_SIZE, + .control_task_stack_size_bytes = CONFIG_RTSP_CONTROL_TASK_STACK_SIZE, + }); + server->set_session_log_level(espp::Logger::Verbosity::WARN); + + auto video_packetizer = std::make_shared( + espp::MjpegPacketizer::Config{.max_payload_size = 1400}); + server->add_track( + espp::RtspServer::TrackConfig{.track_id = video_track_id, .packetizer = video_packetizer}); + + auto audio_packetizer = std::make_shared(espp::GenericPacketizer::Config{ + .max_payload_size = 1400, + .payload_type = 97, + .clock_rate = microphone_sample_rate_hz, + .encoding_name = "L16", + .channels = 1, + .fmtp = "", + .media_type = espp::MediaType::AUDIO, + .log_level = espp::Logger::Verbosity::WARN, + }); + server->add_track( + espp::RtspServer::TrackConfig{.track_id = audio_track_id, .packetizer = audio_packetizer}); + + if (!server->start()) { + logger.error("Failed to start RTSP server on {}:{}", server_address, server_port); + return false; + } + + logger.info("Initializing mDNS"); + auto err = mdns_init(); + if (err != ESP_OK) { + logger.error("Could not initialize mDNS: {}", err); + server.reset(); + return false; + } + + uint8_t mac[6]; + esp_read_mac(mac, ESP_MAC_WIFI_STA); + std::string hostname = fmt::format("xiao-camera-streamer-{:x}{:x}{:x}", mac[3], mac[4], mac[5]); + err = mdns_hostname_set(hostname.c_str()); + if (err != ESP_OK) { + logger.error("Could not set mDNS hostname: {}", err); + mdns_free(); + server.reset(); + return false; + } + logger.info("mDNS hostname set to '{}'", hostname); + + err = mdns_instance_name_set("XIAO Camera Streamer"); + if (err != ESP_OK) { + logger.error("Could not set mDNS instance name: {}", err); + mdns_free(); + server.reset(); + return false; + } + + err = mdns_service_add("RTSP Server", "_rtsp", "_tcp", server_port, nullptr, 0); + if (err != ESP_OK) { + logger.error("Could not add mDNS service: {}", err); + mdns_free(); + server.reset(); + return false; + } + + { + std::lock_guard lock(server_mutex); + rtsp_server = std::move(server); + mdns_started = true; + } + + logger.info("mDNS initialized"); + return true; +} + +void stop_streaming(void) { + std::unique_ptr old_camera_task; + std::unique_ptr old_audio_task; + std::shared_ptr old_rtsp_server; + bool should_stop_mdns = false; + + { + std::lock_guard lock(server_mutex); + old_camera_task = std::move(camera_task); + old_audio_task = std::move(audio_task); + old_rtsp_server = std::move(rtsp_server); + should_stop_mdns = mdns_started; + mdns_started = false; + } + + old_audio_task.reset(); + old_camera_task.reset(); + + if (should_stop_mdns) { + logger.info("Deiniting MDNS"); + mdns_free(); + } + + old_rtsp_server.reset(); +} + +bool camera_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::steady_clock::now(); + + { + std::lock_guard lock(server_mutex); + if (!rtsp_server || !rtsp_server->has_active_sessions()) { + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + auto recommended_capture_period = rtsp_server->get_recommended_capture_period(); + auto capture_cooldown = rtsp_server->get_capture_cooldown(); + if (capture_cooldown > 0ms) { + return wait_until_or_stop(m, cv, task_notified, + start + std::max(recommended_capture_period, capture_cooldown)); + } + } + + auto dma_free = heap_caps_get_free_size(MALLOC_CAP_DMA); + auto dma_largest = heap_caps_get_largest_free_block(MALLOC_CAP_DMA); + if (dma_free < min_dma_free_bytes_for_streaming || + dma_largest < min_dma_largest_block_for_streaming) { + static auto last_pressure_log = std::chrono::steady_clock::time_point{}; + auto now = std::chrono::steady_clock::now(); + if (now - last_pressure_log >= 2s) { + logger.warn("Pausing capture to recover DMA pressure: free={} B, largest_block={} B", + dma_free, dma_largest); + last_pressure_log = now; + } + return wait_until_or_stop(m, cv, task_notified, start + dma_pressure_backoff); + } + + auto *frame_buffer = esp_camera_fb_get(); + if (frame_buffer == nullptr) { + logger.error("Camera capture failed"); + return wait_until_or_stop(m, cv, task_notified, + std::chrono::steady_clock::now() + camera_capture_error_backoff); + } + + if (frame_buffer->len < 2 || frame_buffer->buf[frame_buffer->len - 2] != 0xFF || + frame_buffer->buf[frame_buffer->len - 1] != 0xD9) { + static auto last_bad_frame_log = std::chrono::steady_clock::time_point{}; + auto now = std::chrono::steady_clock::now(); + if (now - last_bad_frame_log >= 2s) { + logger.warn("Dropping camera frame with invalid JPEG end marker"); + last_bad_frame_log = now; + } + esp_camera_fb_return(frame_buffer); + return wait_until_or_stop(m, cv, task_notified, + std::chrono::steady_clock::now() + camera_capture_error_backoff); + } + + std::span jpg_buf(frame_buffer->buf, frame_buffer->len); + { + std::lock_guard lock(server_mutex); + if (rtsp_server) { + rtsp_server->send_frame(video_track_id, jpg_buf); + video_frames_streamed++; + } + } + + esp_camera_fb_return(frame_buffer); + + auto capture_period = target_video_period; + { + std::lock_guard lock(server_mutex); + if (rtsp_server) { + capture_period = std::max(capture_period, rtsp_server->get_recommended_capture_period()); + } + } + return wait_until_or_stop(m, cv, task_notified, start + capture_period); +} + +bool audio_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::steady_clock::now(); + + { + std::lock_guard lock(server_mutex); + if (!rtsp_server || !rtsp_server->has_active_sessions()) { + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + auto capture_cooldown = rtsp_server->get_capture_cooldown(); + if (capture_cooldown > 0ms) { + return wait_until_or_stop(m, cv, task_notified, start + capture_cooldown); + } + } + + if (microphone_rx_channel == nullptr) { + logger.error("Microphone channel is not initialized"); + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + + std::array audio_samples{}; + size_t bytes_read = 0; + auto err = i2s_channel_read(microphone_rx_channel, audio_samples.data(), microphone_frame_bytes, + &bytes_read, audio_capture_timeout.count()); + if (err == ESP_ERR_TIMEOUT) { + return false; + } + if (err != ESP_OK) { + logger.error("Failed reading microphone samples: {} '{}'", err, esp_err_to_name(err)); + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + if (bytes_read == 0) { + return false; + } + + auto *audio_bytes = reinterpret_cast(audio_samples.data()); + { + std::lock_guard lock(server_mutex); + if (rtsp_server) { + std::span audio_frame(audio_bytes, bytes_read); + rtsp_server->send_frame(audio_track_id, audio_frame); + audio_frames_streamed++; + } + } + return false; +} + +bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::steady_clock::now(); + static size_t last_video_frames = 0; + static size_t last_audio_frames = 0; + + auto total_video_frames = video_frames_streamed.load(); + auto total_audio_frames = audio_frames_streamed.load(); + auto delta_video_frames = total_video_frames - last_video_frames; + auto delta_audio_frames = total_audio_frames - last_audio_frames; + last_video_frames = total_video_frames; + last_audio_frames = total_audio_frames; + + logger.info("Frames streamed: video={} (+{}), audio={} (+{})\n{}", total_video_frames, + delta_video_frames, total_audio_frames, delta_audio_frames, + espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA})); + + return wait_until_or_stop(m, cv, task_notified, start + 10s); +} diff --git a/components/xiao-esp32s3-sense/example/partitions.csv b/components/xiao-esp32s3-sense/example/partitions.csv new file mode 100644 index 000000000..419600340 --- /dev/null +++ b/components/xiao-esp32s3-sense/example/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +phy_init, data, phy, 0xf000, 0x1000 +factory, app, factory, 0x10000, 2M +littlefs, data, littlefs, , 1M diff --git a/components/xiao-esp32s3-sense/example/sdkconfig.defaults b/components/xiao-esp32s3-sense/example/sdkconfig.defaults new file mode 100644 index 000000000..c17396c29 --- /dev/null +++ b/components/xiao-esp32s3-sense/example/sdkconfig.defaults @@ -0,0 +1,52 @@ +CONFIG_IDF_TARGET="esp32s3" + +CONFIG_FREERTOS_HZ=1000 + +# on the ESP32S3, which has native USB, we need to set the console so that the +# CLI can be configured correctly: +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y + +# set compiler optimization level to -O2 (compile for performance) +CONFIG_COMPILER_OPTIMIZATION_PERF=y + +CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y +CONFIG_ESPTOOLPY_FLASHSIZE="8MB" +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y + +# Common ESP-related +# +CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096 +CONFIG_ESP_MAIN_TASK_STACK_SIZE=32768 + +# Set esp-timer task stack size to 6KB +CONFIG_ESP_TIMER_TASK_STACK_SIZE=6144 + +# PSRAM configuration +CONFIG_SPIRAM=y +CONFIG_SPIRAM_USE_MALLOC=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y + +# Partition Table +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + +# ESP32-S3 specific tuning for camera + RTSP workloads +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 +CONFIG_LWIP_TCPIP_TASK_PRIO=23 +CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=8 +CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=16 +CONFIG_ESP_WIFI_STATIC_TX_BUFFER=y +CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=16 +CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y +CONFIG_LWIP_TCP_SND_BUF_DEFAULT=16384 +CONFIG_LWIP_TCP_WND_DEFAULT=16384 +CONFIG_LWIP_TCP_RECVMBOX_SIZE=16 +CONFIG_LWIP_UDP_RECVMBOX_SIZE=16 +CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=16 + +# RTSP task stack sizing +CONFIG_RTSP_ACCEPT_TASK_STACK_SIZE=4096 +CONFIG_RTSP_SESSION_TASK_STACK_SIZE=4096 +CONFIG_RTSP_CONTROL_TASK_STACK_SIZE=8192 diff --git a/components/xiao-esp32s3-sense/idf_component.yml b/components/xiao-esp32s3-sense/idf_component.yml new file mode 100644 index 000000000..c435fcb45 --- /dev/null +++ b/components/xiao-esp32s3-sense/idf_component.yml @@ -0,0 +1,29 @@ +## IDF Component Manager Manifest File +license: "MIT" +description: "Seeed Studio XIAO ESP32S3 Sense Board Support Package (BSP) component for ESP-IDF" +url: "https://github.com/esp-cpp/espp/tree/main/components/xiao-esp32s3-sense" +repository: "git://github.com/esp-cpp/espp.git" +maintainers: + - William Emfinger +documentation: "https://esp-cpp.github.io/espp/xiao_esp32s3_sense.html" +examples: + - path: example +tags: + - cpp + - Component + - ESP32S3 + - XIAO + - Sense + - BSP + - Camera + - Microphone + - PDM +dependencies: + idf: + version: '>=5.0' + espp/base_component: '>=1.0' + espp/led: '>=1.0' + espp/math: '>=1.0' + espp/task: '>=1.0' +targets: + - esp32s3 diff --git a/components/xiao-esp32s3-sense/include/xiao-esp32s3-sense.hpp b/components/xiao-esp32s3-sense/include/xiao-esp32s3-sense.hpp new file mode 100644 index 000000000..986a03637 --- /dev/null +++ b/components/xiao-esp32s3-sense/include/xiao-esp32s3-sense.hpp @@ -0,0 +1,332 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "base_component.hpp" +#include "gaussian.hpp" +#include "led.hpp" +#include "task.hpp" + +namespace espp { +/// The XiaoEsp32S3Sense class provides an interface to the Seeed Studio XIAO +/// ESP32S3 Sense board. +/// +/// The class provides access to the following features: +/// - OV2640 camera pin mapping +/// - PDM microphone pin mapping +/// - User LED control / gaussian breathing helpers +/// - User LED pin +/// - microSD SPI pin mapping +/// - Default ESP-IDF PDM RX microphone configuration helpers +/// +/// The class is a singleton and can be accessed using the get() method. +/// +/// \section xiao_esp32s3_sense_example Example +/// \snippet xiao_esp32s3_sense_example.cpp xiao esp32s3 sense example +class XiaoEsp32S3Sense : public BaseComponent { +public: + /// Camera pin mapping for the Sense expansion board. + struct CameraPins { + gpio_num_t pwdn{GPIO_NUM_NC}; ///< Camera power down pin. + gpio_num_t reset{GPIO_NUM_NC}; ///< Camera reset pin. + gpio_num_t xclk{GPIO_NUM_NC}; ///< Camera external clock pin. + gpio_num_t sccb_sda{GPIO_NUM_NC}; ///< Camera SCCB SDA pin. + gpio_num_t sccb_scl{GPIO_NUM_NC}; ///< Camera SCCB SCL pin. + gpio_num_t d7{GPIO_NUM_NC}; ///< Camera D7 / Y9 pin. + gpio_num_t d6{GPIO_NUM_NC}; ///< Camera D6 / Y8 pin. + gpio_num_t d5{GPIO_NUM_NC}; ///< Camera D5 / Y7 pin. + gpio_num_t d4{GPIO_NUM_NC}; ///< Camera D4 / Y6 pin. + gpio_num_t d3{GPIO_NUM_NC}; ///< Camera D3 / Y5 pin. + gpio_num_t d2{GPIO_NUM_NC}; ///< Camera D2 / Y4 pin. + gpio_num_t d1{GPIO_NUM_NC}; ///< Camera D1 / Y3 pin. + gpio_num_t d0{GPIO_NUM_NC}; ///< Camera D0 / Y2 pin. + gpio_num_t vsync{GPIO_NUM_NC}; ///< Camera VSYNC pin. + gpio_num_t href{GPIO_NUM_NC}; ///< Camera HREF pin. + gpio_num_t pclk{GPIO_NUM_NC}; ///< Camera pixel clock pin. + }; + + /// PDM microphone pin mapping for the Sense expansion board. + struct MicrophonePins { + gpio_num_t clk{GPIO_NUM_NC}; ///< PDM clock output pin. + gpio_num_t data{GPIO_NUM_NC}; ///< PDM data input pin. + }; + + /// SPI microSD pin mapping for the Sense expansion board. + struct SdCardPins { + gpio_num_t clk{GPIO_NUM_NC}; ///< SPI clock pin. + gpio_num_t mosi{GPIO_NUM_NC}; ///< SPI MOSI pin. + gpio_num_t miso{GPIO_NUM_NC}; ///< SPI MISO pin. + gpio_num_t cs{GPIO_NUM_NC}; ///< SPI chip-select pin. + }; + + /// Mount point for the microSD card. + static constexpr char mount_point[] = "/sdcard"; + + /// @brief Access the singleton instance of the XiaoEsp32S3Sense class. + /// @return Reference to the singleton instance of the XiaoEsp32S3Sense class. + static XiaoEsp32S3Sense &get() { + static XiaoEsp32S3Sense instance; + return instance; + } + + XiaoEsp32S3Sense(const XiaoEsp32S3Sense &) = delete; + XiaoEsp32S3Sense &operator=(const XiaoEsp32S3Sense &) = delete; + XiaoEsp32S3Sense(XiaoEsp32S3Sense &&) = delete; + XiaoEsp32S3Sense &operator=(XiaoEsp32S3Sense &&) = delete; + + /// Get the documented camera pin mapping. + /// \return The camera pin mapping for the XIAO ESP32S3 Sense. + CameraPins camera_pins() const; + + /// Get the documented PDM microphone pin mapping. + /// \return The microphone pin mapping for the XIAO ESP32S3 Sense. + MicrophonePins microphone_pins() const; + + /// Get the documented SPI microSD pin mapping. + /// \return The microSD pin mapping for the XIAO ESP32S3 Sense. + SdCardPins sd_card_pins() const; + + ///////////////////////////////////////////////////////////////////////////// + // uSD Card + ///////////////////////////////////////////////////////////////////////////// + + /// Configuration for the microSD card. + struct SdCardConfig { + bool format_if_mount_failed = false; ///< Format the card if mount fails. + int max_files = 5; ///< Maximum number of open files. + size_t allocation_unit_size = 2 * 1024; ///< FAT allocation unit size in bytes. + }; + + /// Initialize the microSD card and mount it at mount_point. + /// \return True if the card was initialized and mounted successfully; false + /// otherwise. + bool initialize_sdcard(); + + /// Initialize the microSD card and mount it at mount_point. + /// \param config Mount configuration for the card. + /// \return True if the card was initialized and mounted successfully; false + /// otherwise. + bool initialize_sdcard(const SdCardConfig &config); + + /// Get the mounted microSD card handle. + /// \return Pointer to the mounted card, or `nullptr` if the card has not been + /// initialized successfully. + sdmmc_card_t *sdcard() const { return sdcard_; } + + ///////////////////////////////////////////////////////////////////////////// + // LED + ///////////////////////////////////////////////////////////////////////////// + + /// @brief Initialize the user LED. + /// @param breathing_period The period of the LED breathing effect in seconds. + /// @return True if the LED was successfully initialized, false otherwise. + bool initialize_led(float breathing_period = 3.5f); + + /// @brief Start the user-LED breathing effect. + /// @details Logs an error and does nothing if the LED helper has not been + /// initialized yet. + void start_led_breathing(); + + /// @brief Stop the user-LED breathing effect. + /// @details Logs an error and does nothing if the LED helper has not been + /// initialized yet. + void stop_led_breathing(); + + /// @brief Set the user-LED brightness. + /// @param brightness The brightness of the LED, in the range [0, 1]. + /// @return True if the LED brightness was successfully set, false otherwise. + /// @note The LED brightness can only be set while the breathing task is stopped. + bool set_led_brightness(float brightness); + + /// @brief Get the user-LED brightness. + /// @return The brightness of the LED, in the range [0, 1]. + float get_led_brightness(); + + /// @brief Set the user-LED breathing period. + /// @param period The period of the LED breathing effect in seconds. + /// @return True if the LED breathing period was successfully set, false otherwise. + bool set_led_breathing_period(float period); + + /// @brief Get the user-LED breathing period. + /// @return The period of the LED breathing effect in seconds. + float get_led_breathing_period(); + + /// @brief Get the user LED. + /// @return A shared pointer to the LED. + std::shared_ptr led(); + + /// @brief Get the Gaussian waveform used for the LED breathing effect. + /// @return A reference to the Gaussian waveform. + espp::Gaussian &gaussian(); + + /// Get a default PDM RX clock configuration for the onboard microphone. + /// \param sample_rate_hz The desired microphone sample rate in Hz. + /// \return The default ESP-IDF PDM RX clock configuration. + i2s_pdm_rx_clk_config_t + microphone_clock_config(uint32_t sample_rate_hz = microphone_default_sample_rate_hz()) const; + + /// Get a default PDM RX slot configuration for the onboard microphone. + /// \param slot_mode The microphone slot mode. + /// \return The default ESP-IDF PDM RX slot configuration. + i2s_pdm_rx_slot_config_t + microphone_slot_config(i2s_slot_mode_t slot_mode = I2S_SLOT_MODE_MONO) const; + + /// Get a default PDM RX GPIO configuration for the onboard microphone. + /// \param clock_inverted Whether to invert the generated PDM clock. + /// \return The default ESP-IDF PDM RX GPIO configuration. + i2s_pdm_rx_gpio_config_t microphone_gpio_config(bool clock_inverted = false) const; + + /// Get a complete default PDM RX configuration for the onboard microphone. + /// \param sample_rate_hz The desired microphone sample rate in Hz. + /// \param slot_mode The microphone slot mode. + /// \param clock_inverted Whether to invert the generated PDM clock. + /// \return The default ESP-IDF PDM RX configuration. + i2s_pdm_rx_config_t + microphone_config(uint32_t sample_rate_hz = microphone_default_sample_rate_hz(), + i2s_slot_mode_t slot_mode = I2S_SLOT_MODE_MONO, + bool clock_inverted = false) const; + + /// Get the recommended camera XCLK frequency in Hz. + /// \return The camera XCLK frequency in Hz. + static constexpr int camera_xclk_freq_hz() { return camera_xclk_freq_hz_; } + + /// Get the default microphone sample rate in Hz. + /// \return The default microphone sample rate in Hz. + static constexpr uint32_t microphone_default_sample_rate_hz() { + return microphone_default_sample_rate_hz_; + } + + /// Get the user LED pin on the XIAO ESP32S3 module. + /// \return The user LED GPIO. + static constexpr gpio_num_t user_led_pin() { return user_led_pin_; } + + /// Get the microSD SPI clock pin on the Sense expansion board. + /// \return The microSD SPI clock GPIO. + static constexpr gpio_num_t sd_card_clk_pin() { return sd_card_sck_pin_; } + + /// Get the microSD SPI MOSI pin on the Sense expansion board. + /// \return The microSD SPI MOSI GPIO. + static constexpr gpio_num_t sd_card_mosi_pin() { return sd_card_mosi_pin_; } + + /// Get the microSD SPI MISO pin on the Sense expansion board. + /// \return The microSD SPI MISO GPIO. + static constexpr gpio_num_t sd_card_miso_pin() { return sd_card_miso_pin_; } + + /// Get the default microSD card CS pin on the Sense expansion board. + /// \return The microSD card chip select pin. + static constexpr gpio_num_t sd_card_cs_pin() { return sd_card_cs_pin_; } + + /// Get the camera power down pin. + static constexpr gpio_num_t camera_pwdn_pin() { return camera_pwdn_pin_; } + + /// Get the camera reset pin. + static constexpr gpio_num_t camera_reset_pin() { return camera_reset_pin_; } + + /// Get the camera XCLK pin. + static constexpr gpio_num_t camera_xclk_pin() { return camera_xclk_pin_; } + + /// Get the camera SCCB SDA pin. + static constexpr gpio_num_t camera_sccb_sda_pin() { return camera_sccb_sda_pin_; } + + /// Get the camera SCCB SCL pin. + static constexpr gpio_num_t camera_sccb_scl_pin() { return camera_sccb_scl_pin_; } + + /// Get the camera D7 / Y9 pin. + static constexpr gpio_num_t camera_d7_pin() { return camera_d7_pin_; } + + /// Get the camera D6 / Y8 pin. + static constexpr gpio_num_t camera_d6_pin() { return camera_d6_pin_; } + + /// Get the camera D5 / Y7 pin. + static constexpr gpio_num_t camera_d5_pin() { return camera_d5_pin_; } + + /// Get the camera D4 / Y6 pin. + static constexpr gpio_num_t camera_d4_pin() { return camera_d4_pin_; } + + /// Get the camera D3 / Y5 pin. + static constexpr gpio_num_t camera_d3_pin() { return camera_d3_pin_; } + + /// Get the camera D2 / Y4 pin. + static constexpr gpio_num_t camera_d2_pin() { return camera_d2_pin_; } + + /// Get the camera D1 / Y3 pin. + static constexpr gpio_num_t camera_d1_pin() { return camera_d1_pin_; } + + /// Get the camera D0 / Y2 pin. + static constexpr gpio_num_t camera_d0_pin() { return camera_d0_pin_; } + + /// Get the camera VSYNC pin. + static constexpr gpio_num_t camera_vsync_pin() { return camera_vsync_pin_; } + + /// Get the camera HREF pin. + static constexpr gpio_num_t camera_href_pin() { return camera_href_pin_; } + + /// Get the camera PCLK pin. + static constexpr gpio_num_t camera_pclk_pin() { return camera_pclk_pin_; } + + /// Get the microphone PDM clock pin. + static constexpr gpio_num_t microphone_clk_pin() { return microphone_clk_pin_; } + + /// Get the microphone PDM data pin. + static constexpr gpio_num_t microphone_data_pin() { return microphone_data_pin_; } + +protected: + XiaoEsp32S3Sense(); + float led_breathe(); + bool led_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified); + + static constexpr gpio_num_t camera_pwdn_pin_ = GPIO_NUM_NC; + static constexpr gpio_num_t camera_reset_pin_ = GPIO_NUM_NC; + static constexpr gpio_num_t camera_xclk_pin_ = GPIO_NUM_10; + static constexpr gpio_num_t camera_sccb_sda_pin_ = GPIO_NUM_40; + static constexpr gpio_num_t camera_sccb_scl_pin_ = GPIO_NUM_39; + static constexpr gpio_num_t camera_d7_pin_ = GPIO_NUM_48; + static constexpr gpio_num_t camera_d6_pin_ = GPIO_NUM_11; + static constexpr gpio_num_t camera_d5_pin_ = GPIO_NUM_12; + static constexpr gpio_num_t camera_d4_pin_ = GPIO_NUM_14; + static constexpr gpio_num_t camera_d3_pin_ = GPIO_NUM_16; + static constexpr gpio_num_t camera_d2_pin_ = GPIO_NUM_18; + static constexpr gpio_num_t camera_d1_pin_ = GPIO_NUM_17; + static constexpr gpio_num_t camera_d0_pin_ = GPIO_NUM_15; + static constexpr gpio_num_t camera_vsync_pin_ = GPIO_NUM_38; + static constexpr gpio_num_t camera_href_pin_ = GPIO_NUM_47; + static constexpr gpio_num_t camera_pclk_pin_ = GPIO_NUM_13; + static constexpr int camera_xclk_freq_hz_ = 20 * 1000 * 1000; + + static constexpr gpio_num_t user_led_pin_ = GPIO_NUM_21; + + static constexpr gpio_num_t microphone_clk_pin_ = GPIO_NUM_42; + static constexpr gpio_num_t microphone_data_pin_ = GPIO_NUM_41; + static constexpr uint32_t microphone_default_sample_rate_hz_ = 16 * 1000; + + static constexpr gpio_num_t sd_card_mosi_pin_ = GPIO_NUM_9; + static constexpr gpio_num_t sd_card_miso_pin_ = GPIO_NUM_8; + static constexpr gpio_num_t sd_card_sck_pin_ = GPIO_NUM_7; + static constexpr gpio_num_t sd_card_cs_pin_ = GPIO_NUM_3; + static constexpr spi_host_device_t sd_card_spi_num_ = SPI2_HOST; + static constexpr size_t sd_card_spi_max_transfer_bytes_ = 4 * 1024; + + std::vector led_channels_{{ + .gpio = static_cast(user_led_pin_), + .channel = LEDC_CHANNEL_1, + .timer = LEDC_TIMER_1, + .output_invert = true, + }}; + sdmmc_card_t *sdcard_{nullptr}; + std::shared_ptr led_; + std::unique_ptr led_task_; + std::atomic breathing_period_{3.5f}; + std::chrono::high_resolution_clock::time_point breathing_start_{}; + espp::Gaussian gaussian_{{.gamma = 0.1f, .alpha = 1.0f, .beta = 0.5f}}; +}; // class XiaoEsp32S3Sense +} // namespace espp diff --git a/components/xiao-esp32s3-sense/src/xiao-esp32s3-sense.cpp b/components/xiao-esp32s3-sense/src/xiao-esp32s3-sense.cpp new file mode 100644 index 000000000..4e77356b5 --- /dev/null +++ b/components/xiao-esp32s3-sense/src/xiao-esp32s3-sense.cpp @@ -0,0 +1,235 @@ +#include "xiao-esp32s3-sense.hpp" + +#include + +#include + +using namespace espp; + +XiaoEsp32S3Sense::XiaoEsp32S3Sense() + : BaseComponent("XiaoEsp32S3Sense") {} + +XiaoEsp32S3Sense::CameraPins XiaoEsp32S3Sense::camera_pins() const { + return { + .pwdn = camera_pwdn_pin(), + .reset = camera_reset_pin(), + .xclk = camera_xclk_pin(), + .sccb_sda = camera_sccb_sda_pin(), + .sccb_scl = camera_sccb_scl_pin(), + .d7 = camera_d7_pin(), + .d6 = camera_d6_pin(), + .d5 = camera_d5_pin(), + .d4 = camera_d4_pin(), + .d3 = camera_d3_pin(), + .d2 = camera_d2_pin(), + .d1 = camera_d1_pin(), + .d0 = camera_d0_pin(), + .vsync = camera_vsync_pin(), + .href = camera_href_pin(), + .pclk = camera_pclk_pin(), + }; +} + +XiaoEsp32S3Sense::MicrophonePins XiaoEsp32S3Sense::microphone_pins() const { + return { + .clk = microphone_clk_pin(), + .data = microphone_data_pin(), + }; +} + +XiaoEsp32S3Sense::SdCardPins XiaoEsp32S3Sense::sd_card_pins() const { + return { + .clk = sd_card_clk_pin(), + .mosi = sd_card_mosi_pin(), + .miso = sd_card_miso_pin(), + .cs = sd_card_cs_pin(), + }; +} + +bool XiaoEsp32S3Sense::initialize_sdcard() { return initialize_sdcard(SdCardConfig{}); } + +bool XiaoEsp32S3Sense::initialize_sdcard(const XiaoEsp32S3Sense::SdCardConfig &config) { + if (sdcard_) { + logger_.error("SD card already initialized"); + return false; + } + + logger_.info("Initializing microSD card"); + + esp_vfs_fat_sdmmc_mount_config_t mount_config{}; + mount_config.format_if_mount_failed = config.format_if_mount_failed; + mount_config.max_files = config.max_files; + mount_config.allocation_unit_size = config.allocation_unit_size; + + spi_bus_config_t bus_config{}; + bus_config.mosi_io_num = sd_card_mosi_pin(); + bus_config.miso_io_num = sd_card_miso_pin(); + bus_config.sclk_io_num = sd_card_clk_pin(); + bus_config.quadwp_io_num = -1; + bus_config.quadhd_io_num = -1; + bus_config.max_transfer_sz = sd_card_spi_max_transfer_bytes_; + + esp_err_t ret = spi_bus_initialize(sd_card_spi_num_, &bus_config, SDSPI_DEFAULT_DMA); + if (ret != ESP_OK) { + logger_.error("Failed to initialize microSD SPI bus: {}", esp_err_to_name(ret)); + return false; + } + + sdmmc_host_t host = SDSPI_HOST_DEFAULT(); + host.slot = sd_card_spi_num_; + + sdspi_device_config_t slot_config = SDSPI_DEVICE_CONFIG_DEFAULT(); + slot_config.gpio_cs = sd_card_cs_pin(); + slot_config.host_id = sd_card_spi_num_; + + ret = esp_vfs_fat_sdspi_mount(mount_point, &host, &slot_config, &mount_config, &sdcard_); + if (ret != ESP_OK) { + if (ret == ESP_FAIL) { + logger_.error("Failed to mount microSD filesystem"); + } else { + logger_.error("Failed to initialize microSD card ({}). Make sure the card is inserted and " + "the bus has pull-ups.", + esp_err_to_name(ret)); + } + spi_bus_free(sd_card_spi_num_); + sdcard_ = nullptr; + return false; + } + + logger_.info("microSD filesystem mounted at {}", mount_point); + sdmmc_card_print_info(stdout, sdcard_); + return true; +} + +bool XiaoEsp32S3Sense::initialize_led(float breathing_period) { + if (led_) { + logger_.error("LED already initialized"); + return false; + } + led_ = std::make_shared(espp::Led::Config{ + .timer = LEDC_TIMER_1, + .frequency_hz = 5000, + .channels = led_channels_, + .duty_resolution = LEDC_TIMER_10_BIT, + }); + using namespace std::placeholders; + led_task_ = espp::Task::make_unique( + {.callback = std::bind(&XiaoEsp32S3Sense::led_task_callback, this, _1, _2, _3), + .task_config = {.name = "xiao_led_breathe"}}); + if (!led_task_) { + logger_.error("Could not create LED breathing task"); + led_.reset(); + return false; + } + if (!set_led_breathing_period(breathing_period)) { + led_task_.reset(); + led_.reset(); + return false; + } + return true; +} + +void XiaoEsp32S3Sense::start_led_breathing() { + if (led_ == nullptr || led_task_ == nullptr) { + logger_.error("LED not initialized"); + return; + } + breathing_start_ = std::chrono::high_resolution_clock::now(); + led_task_->start(); +} + +void XiaoEsp32S3Sense::stop_led_breathing() { + if (led_ == nullptr || led_task_ == nullptr) { + logger_.error("LED not initialized"); + return; + } + led_task_->stop(); +} + +bool XiaoEsp32S3Sense::set_led_brightness(float brightness) { + if (led_ == nullptr || led_task_ == nullptr) { + logger_.error("LED not initialized"); + return false; + } + if (led_task_->is_running()) { + logger_.error("Cannot set brightness while breathing"); + return false; + } + brightness = std::clamp(brightness, 0.0f, 1.0f); + led_->set_duty(led_channels_[0].channel, 100.0f * brightness); + return true; +} + +float XiaoEsp32S3Sense::get_led_brightness() { + if (led_ == nullptr) { + logger_.error("LED not initialized"); + return 0.0f; + } + auto maybe_duty = led_->get_duty(led_channels_[0].channel); + if (!maybe_duty.has_value()) { + logger_.error("Failed to get LED duty"); + return 0.0f; + } + return maybe_duty.value() / 100.0f; +} + +bool XiaoEsp32S3Sense::set_led_breathing_period(float breathing_period) { + if (breathing_period <= 0.0f) { + logger_.error("Invalid breathing period: {}", breathing_period); + return false; + } + breathing_period_ = breathing_period; + return true; +} + +float XiaoEsp32S3Sense::get_led_breathing_period() { return breathing_period_; } + +std::shared_ptr XiaoEsp32S3Sense::led() { return led_; } + +espp::Gaussian &XiaoEsp32S3Sense::gaussian() { return gaussian_; } + +float XiaoEsp32S3Sense::led_breathe() { + auto now = std::chrono::high_resolution_clock::now(); + auto elapsed = std::chrono::duration(now - breathing_start_).count(); + float t = std::fmod(elapsed, breathing_period_) / breathing_period_; + return gaussian_(t); +} + +bool XiaoEsp32S3Sense::led_task_callback(std::mutex &m, std::condition_variable &cv, + bool &task_notified) { + using namespace std::chrono_literals; + led_->set_duty(led_channels_[0].channel, 100.0f * led_breathe()); + std::unique_lock lk(m); + cv.wait_for(lk, 10ms, [&task_notified] { return task_notified; }); + task_notified = false; + return false; +} + +i2s_pdm_rx_clk_config_t XiaoEsp32S3Sense::microphone_clock_config(uint32_t sample_rate_hz) const { + i2s_pdm_rx_clk_config_t config = I2S_PDM_RX_CLK_DEFAULT_CONFIG(sample_rate_hz); + return config; +} + +i2s_pdm_rx_slot_config_t XiaoEsp32S3Sense::microphone_slot_config(i2s_slot_mode_t slot_mode) const { + i2s_pdm_rx_slot_config_t config = + I2S_PDM_RX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, slot_mode); + return config; +} + +i2s_pdm_rx_gpio_config_t XiaoEsp32S3Sense::microphone_gpio_config(bool clock_inverted) const { + i2s_pdm_rx_gpio_config_t config = {}; + config.clk = microphone_clk_pin(); + config.din = microphone_data_pin(); + config.invert_flags.clk_inv = clock_inverted; + return config; +} + +i2s_pdm_rx_config_t XiaoEsp32S3Sense::microphone_config(uint32_t sample_rate_hz, + i2s_slot_mode_t slot_mode, + bool clock_inverted) const { + return { + .clk_cfg = microphone_clock_config(sample_rate_hz), + .slot_cfg = microphone_slot_config(slot_mode), + .gpio_cfg = microphone_gpio_config(clock_inverted), + }; +} diff --git a/doc/Doxyfile b/doc/Doxyfile index bc336c5eb..c739531b3 100755 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -166,7 +166,8 @@ EXAMPLE_PATH = \ $(PROJECT_PATH)/components/wrover-kit/example/main/wrover_kit_example.cpp \ $(PROJECT_PATH)/components/ws-s3-geek/example/main/ws_s3_geek_example.cpp \ $(PROJECT_PATH)/components/ws-s3-lcd-1-47/example/main/ws_s3_lcd_1_47_example.cpp \ - $(PROJECT_PATH)/components/ws-s3-touch/example/main/ws_s3_touch_example.cpp + $(PROJECT_PATH)/components/ws-s3-touch/example/main/ws_s3_touch_example.cpp \ + $(PROJECT_PATH)/components/xiao-esp32s3-sense/example/main/xiao_esp32s3_sense_example.cpp ## The 'INPUT' statement below is used as input by script 'gen-df-input.py' ## to automatically generate API reference list files header_file.inc @@ -373,7 +374,8 @@ INPUT = \ $(PROJECT_PATH)/components/wrover-kit/include/wrover-kit.hpp \ $(PROJECT_PATH)/components/ws-s3-geek/include/ws-s3-geek.hpp \ $(PROJECT_PATH)/components/ws-s3-lcd-1-47/include/ws-s3-lcd-1-47.hpp \ - $(PROJECT_PATH)/components/ws-s3-touch/include/ws-s3-touch.hpp + $(PROJECT_PATH)/components/ws-s3-touch/include/ws-s3-touch.hpp \ + $(PROJECT_PATH)/components/xiao-esp32s3-sense/include/xiao-esp32s3-sense.hpp ## Get warnings for functions that have no documentation for their parameters or return value ## diff --git a/doc/en/xiao_esp32s3_sense.rst b/doc/en/xiao_esp32s3_sense.rst new file mode 100644 index 000000000..eb2d2c0eb --- /dev/null +++ b/doc/en/xiao_esp32s3_sense.rst @@ -0,0 +1,28 @@ +XIAO ESP32S3 Sense +****************** + +XIAO ESP32S3 Sense +------------------ + +The Seeed Studio XIAO ESP32S3 Sense combines the XIAO ESP32S3 module with the +Sense expansion board, which adds an OV2640 camera, a PDM microphone, and a +microSD card slot. + +The `espp::XiaoEsp32S3Sense` component provides a singleton hardware abstraction +for the board's documented camera, microphone, user-LED, and microSD pin +mappings, as well as helpers for building the default ESP-IDF PDM RX microphone +configuration, mounting the onboard microSD card over SDSPI, and controlling the +onboard LED with a Gaussian breathing effect. + +.. ------------------------------- Example ------------------------------------- + +.. toctree:: + + xiao_esp32s3_sense_example.md + +.. ---------------------------- API Reference ---------------------------------- + +API Reference +------------- + +.. include-build-file:: inc/xiao-esp32s3-sense.inc diff --git a/doc/en/xiao_esp32s3_sense_example.md b/doc/en/xiao_esp32s3_sense_example.md new file mode 100644 index 000000000..c75931655 --- /dev/null +++ b/doc/en/xiao_esp32s3_sense_example.md @@ -0,0 +1,2 @@ +```{include} ../../components/xiao-esp32s3-sense/example/README.md +```