diff --git a/components/esp32-timer-cam/example/CMakeLists.txt b/components/esp32-timer-cam/example/CMakeLists.txt index f3aafadb8..ef7664675 100644 --- a/components/esp32-timer-cam/example/CMakeLists.txt +++ b/components/esp32-timer-cam/example/CMakeLists.txt @@ -2,17 +2,27 @@ # in this exact order for cmake to work correctly cmake_minimum_required(VERSION 3.20) -set(ENV{IDF_COMPONENT_MANAGER} "0") include($ENV{IDF_PATH}/tools/cmake/project.cmake) # add the component directories that we want to use set(EXTRA_COMPONENT_DIRS - "../../../components/" + "../../../components/base_component" + "../../../components/cli" + "../../../components/esp32-timer-cam" + "../../../components/format" + "../../../components/i2c" + "../../../components/logger" + "../../../components/monitor" + "../../../components/nvs" + "../../../components/rtsp" + "../../../components/socket" + "../../../components/task" + "../../../components/wifi" ) set( COMPONENTS - "main esptool_py esp32-timer-cam" + "main esptool_py esp32-camera esp32-timer-cam mdns monitor nvs rtsp socket task wifi" CACHE STRING "List of components to include" ) diff --git a/components/esp32-timer-cam/example/main/Kconfig.projbuild b/components/esp32-timer-cam/example/main/Kconfig.projbuild new file mode 100644 index 000000000..a60526ad6 --- /dev/null +++ b/components/esp32-timer-cam/example/main/Kconfig.projbuild @@ -0,0 +1,27 @@ +menu "Camera Streamer Configuration" + + config RTSP_SERVER_PORT + int "RTSP Server Port" + default 8554 + help + The port number of the RTSP server. + + config ESP_WIFI_SSID + string "WiFi SSID" + default "" + help + SSID (network name) for the camera streamer to connect to. + + config ESP_WIFI_PASSWORD + string "WiFi Password" + default "" + help + WiFi password (WPA or WPA2) for the camera streamer to use. + + config ESP_MAXIMUM_RETRY + int "Maximum retry" + default 5 + help + Set the Maximum retry to avoid station reconnecting to the AP unlimited when the AP is really inexistent. + +endmenu diff --git a/components/esp32-timer-cam/example/main/esp_timer_cam_example.cpp b/components/esp32-timer-cam/example/main/esp_timer_cam_example.cpp index 72b8b19b4..d4e55e351 100644 --- a/components/esp32-timer-cam/example/main/esp_timer_cam_example.cpp +++ b/components/esp32-timer-cam/example/main/esp_timer_cam_example.cpp @@ -1,43 +1,400 @@ +#include "sdkconfig.h" + #include -#include -#include -#include +#include +#include + +#include +#include + +#include "cli.hpp" #include "esp32-timer-cam.hpp" +#include "heap_monitor.hpp" +#include "nvs.hpp" +#include "task.hpp" +#include "task_monitor.hpp" +#include "tcp_socket.hpp" +#include "udp_socket.hpp" +#include "wifi_sta.hpp" +#include "wifi_sta_menu.hpp" + +#include "esp_camera.h" + +#include "rtsp_server.hpp" using namespace std::chrono_literals; +static espp::Logger logger({.tag = "Camera Streamer", .level = espp::Logger::Verbosity::INFO}); + +namespace { +constexpr auto idle_capture_poll_period = 250ms; +constexpr auto dma_pressure_backoff = 250ms; +constexpr auto target_stream_period = 100ms; +constexpr size_t min_dma_free_bytes_for_streaming = 12 * 1024; +constexpr size_t min_dma_largest_block_for_streaming = 4 * 1024; +} // namespace + +std::recursive_mutex server_mutex; +std::unique_ptr camera_task; +std::unique_ptr memory_monitor_task; +std::unique_ptr task_monitor; +std::shared_ptr rtsp_server; +std::atomic frames_streamed{0}; + +esp_err_t initialize_camera(void); +bool start_rtsp_server(std::string_view server_address, int server_port); +bool camera_task_fn(std::mutex &m, std::condition_variable &cv); +bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); + extern "C" void app_main(void) { - espp::Logger logger({.tag = "ESP32 TimerCam Example", .level = espp::Logger::Verbosity::INFO}); - logger.info("Starting example!"); + esp_err_t err; + logger.info("Bootup"); //! [esp timer cam example] - espp::EspTimerCam &timer_cam = espp::EspTimerCam::get(); + auto &timer_cam = espp::EspTimerCam::get(); - // initialize the LED - static constexpr float led_breathing_period = 3.5f; - if (!timer_cam.initialize_led(led_breathing_period)) { - logger.error("Failed to initialize LED!"); + // initialize RTC + if (!timer_cam.initialize_rtc()) { + logger.error("Could not initialize RTC"); return; } - // initialize the rtc - if (!timer_cam.initialize_rtc()) { - logger.error("Failed to initialize RTC!"); + // initialize LED + static constexpr float disconnected_led_breathing_period = 1.0f; + static constexpr float connected_led_breathing_period = 3.5f; + if (!timer_cam.initialize_led(disconnected_led_breathing_period)) { + logger.error("Could not initialize LED"); return; } - - // start the LED breathing timer_cam.start_led_breathing(); + //! [esp timer cam example] - // loop forever - while (true) { - // print out the battery voltage - logger.info("Battery voltage: {:.02f} V", timer_cam.get_battery_voltage()); - std::this_thread::sleep_for(100ms); - // go up a line to overwrite the previous message - logger.move_up(); - logger.clear_line(); + // initialize camera + logger.info("Initializing camera"); + err = initialize_camera(); + if (err != ESP_OK) { + logger.error("Could not initialize camera: {} '{}'", err, esp_err_to_name(err)); + return; } - //! [esp timer cam example] + + logger.info("Starting memory monitors"); + task_monitor = std::make_unique(espp::TaskMonitor::Config{.period = 30s}); + memory_monitor_task = espp::Task::make_unique(espp::Task::Config{ + .callback = memory_monitor_task_fn, + .task_config = + { + .name = "Memory Monitor", + .stack_size_bytes = 6 * 1024, + }, + .log_level = espp::Logger::Verbosity::WARN, + }); + + // initialize WiFi + logger.info("Initializing WiFi"); + espp::WifiSta wifi_sta( + {.ssid = CONFIG_ESP_WIFI_SSID, + .password = CONFIG_ESP_WIFI_PASSWORD, + .num_connect_retries = CONFIG_ESP_MAXIMUM_RETRY, + .on_connected = + []() { + static auto &timer_cam = espp::EspTimerCam::get(); + timer_cam.set_led_breathing_period(connected_led_breathing_period); + memory_monitor_task->start(); + }, + .on_disconnected = + []() { + logger.info("WiFi disconnected, stopping RTSP streaming"); + memory_monitor_task->stop(); + static auto &timer_cam = espp::EspTimerCam::get(); + timer_cam.set_led_breathing_period(disconnected_led_breathing_period); + std::lock_guard lock(server_mutex); + logger.info("Stopping camera task"); + camera_task.reset(); + logger.info("Stopping RTSP server"); + rtsp_server.reset(); + logger.info("Deiniting MDNS"); + mdns_free(); + }, + .on_got_ip = + [](ip_event_got_ip_t *eventdata) { + auto server_address = fmt::format("{}.{}.{}.{}", IP2STR(&eventdata->ip_info.ip)); + logger.info("got IP: {}", server_address); + // create the camera and rtsp server, and the cv/m + // they'll use to communicate + std::lock_guard lock(server_mutex); + if (!start_rtsp_server(server_address, CONFIG_RTSP_SERVER_PORT)) { + logger.error("RTSP server failed to start, not starting camera task"); + return; + } + // initialize the camera + logger.info("Creating camera task"); + camera_task = espp::Task::make_unique( + espp::Task::Config{.callback = camera_task_fn, + .task_config = {.name = "Camera Task", .priority = 10}}); + camera_task->start(); + }}); + + if (esp_wifi_set_ps(WIFI_PS_NONE) != ESP_OK) { + logger.warn("Could not disable WiFi power save; RTSP streaming may hit TX backpressure"); + } else { + logger.info("Disabled WiFi power save for RTSP streaming"); + } + + espp::WifiStaMenu sta_menu(wifi_sta); + auto root_menu = sta_menu.get(); + root_menu->Insert( + "memory", + [](std::ostream &out) { + out << "Frames streamed: " << frames_streamed.load() << std::endl; + out << espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA}) + << std::endl; + }, + "Display current heap monitor information."); + root_menu->Insert( + "battery", + [](std::ostream &out) { + static auto &timer_cam = espp::EspTimerCam::get(); + out << fmt::format("Battery voltage: {:.2f}\n", timer_cam.get_battery_voltage()); + }, + "Display the current battery voltage."); + + cli::Cli cli(std::move(root_menu)); + cli::SetColor(); + cli.ExitAction([](auto &out) { out << "Goodbye and thanks for all the fish.\n"; }); + + espp::Cli input(cli); + input.SetInputHistorySize(10); + input.Start(); +} + +esp_err_t initialize_camera(void) { + /** + * @note display sizes supported: + * * QVGA: 320x240 + * * WQVGA: 400x240 + * * HVGA: 480x320 + * * VGA: 640x480 + * * WVGA: 768x480 + * * FWVGA: 854x480 + * * SVGA: 800x600 + * * DVGA: 960x640 + * * WSVGA: 1024x600 + * * XGA: 1024x768 + * * WXGA: 1280x800 + * * WSXGA: 1440x900 + * * SXGA: 1280x1024 + * * UXGA: 1600x1200 + */ + + auto &timer_cam = espp::EspTimerCam::get(); + static camera_config_t camera_config = { + .pin_pwdn = -1, + .pin_reset = timer_cam.get_camera_reset_pin(), + .pin_xclk = timer_cam.get_camera_xclk_pin(), + .pin_sccb_sda = timer_cam.get_camera_sda_pin(), + .pin_sccb_scl = timer_cam.get_camera_scl_pin(), + + .pin_d7 = timer_cam.get_camera_d7_pin(), + .pin_d6 = timer_cam.get_camera_d6_pin(), + .pin_d5 = timer_cam.get_camera_d5_pin(), + .pin_d4 = timer_cam.get_camera_d4_pin(), + .pin_d3 = timer_cam.get_camera_d3_pin(), + .pin_d2 = timer_cam.get_camera_d2_pin(), + .pin_d1 = timer_cam.get_camera_d1_pin(), + .pin_d0 = timer_cam.get_camera_d0_pin(), + .pin_vsync = timer_cam.get_camera_vsync_pin(), + .pin_href = timer_cam.get_camera_href_pin(), + .pin_pclk = timer_cam.get_camera_pclk_pin(), + + // EXPERIMENTAL: Set to 16MHz on ESP32-S2 or ESP32-S3 to enable EDMA mode + .xclk_freq_hz = timer_cam.get_camera_xclk_freq_hz(), + .ledc_timer = LEDC_TIMER_0, + .ledc_channel = LEDC_CHANNEL_0, + + .pixel_format = PIXFORMAT_JPEG, // YUV422,GRAYSCALE,RGB565,JPEG + .frame_size = FRAMESIZE_QVGA, // QVGA-UXGA, For ESP32, do not use sizes above QVGA when not + // JPEG. The performance of the ESP32-S series has improved a + // lot, but JPEG mode always gives better frame rates. + + .jpeg_quality = 15, // 0-63, for OV series camera sensors, lower number means higher quality + .fb_count = 2, // When jpeg mode is used, if fb_count more than one, the driver will work in + // continuous mode. + .fb_location = CAMERA_FB_IN_PSRAM, + .grab_mode = + CAMERA_GRAB_LATEST, // CAMERA_GRAB_WHEN_EMPTY // . Sets when buffers should be filled + .sccb_i2c_port = I2C_NUM_0}; + auto err = esp_camera_init(&camera_config); + if (err != ESP_OK) { + logger.error("Could not initialize camera: {} '{}'", err, esp_err_to_name(err)); + return err; + } + // set the mirror and flip - specific to the ESP32-TimerCam! + logger.info("Enabling camera vflip"); + sensor_t *s = esp_camera_sensor_get(); + s->set_vflip(s, true); + s->set_hmirror(s, false); + return ESP_OK; +} + +bool start_rtsp_server(std::string_view server_address, int server_port) { + logger.info("Creating RTSP server at {}:{}", server_address, server_port); + std::lock_guard lock(server_mutex); + rtsp_server = std::make_shared( + espp::RtspServer::Config{.server_address = std::string(server_address), + .port = server_port, + .path = "mjpeg/1", + .log_level = espp::Logger::Verbosity::WARN}); + rtsp_server->set_session_log_level(espp::Logger::Verbosity::WARN); + if (!rtsp_server->start()) { + logger.error("Failed to start RTSP server on {}:{}", server_address, server_port); + rtsp_server.reset(); + return false; + } + + // initialize mDNS + logger.info("Initializing mDNS"); + esp_err_t err = mdns_init(); + if (err != ESP_OK) { + logger.error("Could not initialize mDNS: {}", err); + rtsp_server.reset(); + return false; + } + + uint8_t mac[6]; + esp_read_mac(mac, ESP_MAC_WIFI_STA); + std::string hostname = fmt::format("camera-streamer-{:x}{:x}{:x}", mac[3], mac[4], mac[5]); + err = mdns_hostname_set(hostname.c_str()); + if (err != ESP_OK) { + logger.error("Could not set mDNS hostname: {}", err); + mdns_free(); + rtsp_server.reset(); + return false; + } + logger.info("mDNS hostname set to '{}'", hostname); + err = mdns_instance_name_set("Camera Streamer"); + if (err != ESP_OK) { + logger.error("Could not set mDNS instance name: {}", err); + mdns_free(); + rtsp_server.reset(); + return false; + } + err = mdns_service_add("RTSP Server", "_rtsp", "_tcp", server_port, NULL, 0); + if (err != ESP_OK) { + logger.error("Could not add mDNS service: {}", err); + mdns_free(); + rtsp_server.reset(); + return false; + } + logger.info("mDNS initialized"); + return true; +} + +bool camera_task_fn(std::mutex &m, std::condition_variable &cv) { + auto start = std::chrono::high_resolution_clock::now(); + auto wait_until = [&](auto deadline) { + std::unique_lock lk(m); + cv.wait_until(lk, deadline); + }; + std::shared_ptr server_snapshot; + + { + std::lock_guard lock(server_mutex); + server_snapshot = rtsp_server; + if (!server_snapshot || !server_snapshot->has_active_sessions()) { + wait_until(start + idle_capture_poll_period); + return false; + } + auto recommended_capture_period = server_snapshot->get_recommended_capture_period(); + auto capture_cooldown = server_snapshot->get_capture_cooldown(); + if (capture_cooldown > 0ms) { + wait_until(start + std::max(recommended_capture_period, capture_cooldown)); + return false; + } + } + + auto dma_free = heap_caps_get_free_size(MALLOC_CAP_DMA); + auto dma_largest = heap_caps_get_largest_free_block(MALLOC_CAP_DMA); + if (dma_free < min_dma_free_bytes_for_streaming || + dma_largest < min_dma_largest_block_for_streaming) { + static auto last_pressure_log = std::chrono::steady_clock::time_point{}; + auto now = std::chrono::steady_clock::now(); + if (now - last_pressure_log >= 2s) { + logger.warn("Pausing capture to recover DMA pressure: free={} B, largest_block={} B", + dma_free, dma_largest); + last_pressure_log = now; + } + wait_until(start + dma_pressure_backoff); + return false; + } + + // take image + static camera_fb_t *fb = NULL; + static size_t _jpg_buf_len; + static uint8_t *_jpg_buf; + + fb = esp_camera_fb_get(); + if (!fb) { + logger.error("Camera capture failed"); + return false; + } + + _jpg_buf_len = fb->len; + _jpg_buf = fb->buf; + + if (_jpg_buf_len < 2 || _jpg_buf[_jpg_buf_len - 2] != 0xFF || + _jpg_buf[_jpg_buf_len - 1] != 0xD9) { + esp_camera_fb_return(fb); + return false; + } + + { + std::lock_guard lock(server_mutex); + server_snapshot = rtsp_server; + } + if (server_snapshot) { + std::span jpg_buf(_jpg_buf, _jpg_buf_len); + server_snapshot->send_frame(jpg_buf); + frames_streamed++; + } + + esp_camera_fb_return(fb); + + // sleep for a short period to target ~10 FPS to yield to other tasks. + { + auto capture_period = target_stream_period; + { + std::lock_guard lock(server_mutex); + server_snapshot = rtsp_server; + if (server_snapshot) { + capture_period = + std::max(capture_period, server_snapshot->get_recommended_capture_period()); + } + } + wait_until(start + capture_period); + } + + return false; +}; + +bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::high_resolution_clock::now(); + static size_t last_frames_streamed = 0; + auto total_frames = frames_streamed.load(); + auto delta_frames = total_frames - last_frames_streamed; + last_frames_streamed = total_frames; + logger.info("Frames streamed: {} (+{} in last interval)\n{}", total_frames, delta_frames, + espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA})); + { + std::unique_lock lk(m); + auto stop_requested = + cv.wait_until(lk, start + 10s, [&task_notified] { return task_notified; }); + task_notified = false; + if (stop_requested) { + return true; + } + } + return false; } diff --git a/components/esp32-timer-cam/example/main/idf_component.yml b/components/esp32-timer-cam/example/main/idf_component.yml new file mode 100644 index 000000000..f7867c161 --- /dev/null +++ b/components/esp32-timer-cam/example/main/idf_component.yml @@ -0,0 +1,14 @@ +## IDF Component Manager Manifest File +dependencies: + ## Required IDF version + idf: + version: '>=5.0' + espressif/mdns: '>=1.8' + espressif/esp32-camera: '>=2.0' + espp/rtsp: '>=1.0' + espp/wifi: '>=1.0' + espp/monitor: '>=1.0' + espp/socket: '>=1.0' + espp/nvs: '>=1.0' + espp/task: '>=1.0' + espp/esp32-timer-cam: '>=1.0' diff --git a/components/esp32-timer-cam/example/partitions.csv b/components/esp32-timer-cam/example/partitions.csv new file mode 100644 index 000000000..419600340 --- /dev/null +++ b/components/esp32-timer-cam/example/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +phy_init, data, phy, 0xf000, 0x1000 +factory, app, factory, 0x10000, 2M +littlefs, data, littlefs, , 1M diff --git a/components/esp32-timer-cam/example/sdkconfig.defaults b/components/esp32-timer-cam/example/sdkconfig.defaults index 1597592fb..6daae9bca 100644 --- a/components/esp32-timer-cam/example/sdkconfig.defaults +++ b/components/esp32-timer-cam/example/sdkconfig.defaults @@ -1,22 +1,54 @@ CONFIG_IDF_TARGET="esp32" -CONFIG_FREERTOS_HZ=1000 - -# set compiler optimization level to -O2 (compile for performance) CONFIG_COMPILER_OPTIMIZATION_PERF=y +# CONFIG_COMPILER_OPTIMIZATION_SIZE=y CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y CONFIG_ESPTOOLPY_FLASHSIZE="4MB" +CONFIG_FREERTOS_HZ=1000 + +# Common ESP-related +# +CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096 +CONFIG_ESP_MAIN_TASK_STACK_SIZE=32768 + +# SPIRAM Configuration +CONFIG_SPIRAM=y +CONFIG_SPIRAM_USE_MALLOC=y +# CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +# CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y + +# +# Partition Table +# +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + + # ESP32-specific # CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 -# Common ESP-related +# ESP32-Camera specific +# CONFIG_SCCB_HARDWARE_I2C_DRIVER_LEGACY=y + +# CONFIG_LWIP_IRAM_OPTIMIZATION=y +CONFIG_LWIP_TCPIP_TASK_PRIO=23 + # -CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096 -CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384 +# ESP32S3-specific +# +CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=8 +CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=16 +CONFIG_ESP_WIFI_STATIC_TX_BUFFER=y +CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=16 +CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y -# Set esp-timer task stack size to 6KB -CONFIG_ESP_TIMER_TASK_STACK_SIZE=6144 +CONFIG_LWIP_TCP_SND_BUF_DEFAULT=16384 +CONFIG_LWIP_TCP_WND_DEFAULT=16384 +CONFIG_LWIP_TCP_RECVMBOX_SIZE=16 +CONFIG_LWIP_UDP_RECVMBOX_SIZE=16 +CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=16