feat: support camera capture to livekit

main/boards/common/camera.h

@@ -7,6 +7,8 @@ class Camera {
 public:
     virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0;
     virtual bool Capture() = 0;
+    virtual bool CaptureBackground() { return Capture(); }
+    virtual bool CaptureToJpeg(std::string& jpeg_data, bool show_preview = false) { return false; }
     virtual bool SetHMirror(bool enabled) = 0;
     virtual bool SetVFlip(bool enabled) = 0;
     virtual bool SetSwapBytes(bool enabled) { return false; }  // Optional, default no-op

main/boards/common/esp_video.cc

@@ -24,6 +24,7 @@
 #include "lvgl_display.h"
 #include "mcp_server.h"
 #include "system_info.h"
+#include "esp_timer.h"
 
 #ifdef CONFIG_XIAOZHI_ENABLE_CAMERA_DEBUG_MODE
 #undef LOG_LOCAL_LEVEL
@@ -55,6 +56,7 @@
 
 
 #define TAG "EspVideo"
+#define FOREGROUND_CAPTURE_PROTECTION_US (10 * 1000 * 1000)
 
 #if defined(CONFIG_CAMERA_SENSOR_SWAP_PIXEL_BYTE_ORDER) || defined(CONFIG_XIAOZHI_ENABLE_CAMERA_ENDIANNESS_SWAP)
 #warning \
@@ -381,11 +383,47 @@ EspVideo::~EspVideo() {
 }
 
 void EspVideo::SetExplainUrl(const std::string& url, const std::string& token) {
+    std::lock_guard<std::mutex> lock(frame_mutex_);
     explain_url_ = url;
     explain_token_ = token;
 }
 
 bool EspVideo::Capture() {
+    return CaptureFrame(true);
+}
+
+bool EspVideo::CaptureBackground() {
+    return CaptureFrame(false);
+}
+
+bool EspVideo::CaptureToJpeg(std::string& jpeg_data, bool show_preview) {
+    jpeg_data.clear();
+    if (!CaptureFrame(show_preview)) {
+        return false;
+    }
+
+    std::lock_guard<std::mutex> lock(frame_mutex_);
+    if (frame_.data == nullptr || frame_.len == 0) {
+        return false;
+    }
+
+    uint16_t w = frame_.width ? frame_.width : 320;
+    uint16_t h = frame_.height ? frame_.height : 240;
+    return image_to_jpeg_cb(
+        frame_.data, frame_.len, w, h, frame_.format, 60,
+        [](void* arg, size_t index, const void* data, size_t len) -> size_t {
+            auto jpeg_data = static_cast<std::string*>(arg);
+            if (data != nullptr && len > 0) {
+                jpeg_data->append(static_cast<const char*>(data), len);
+            }
+            return len;
+        },
+        &jpeg_data);
+}
+
+bool EspVideo::CaptureFrame(bool show_preview) {
+    std::lock_guard<std::mutex> lock(frame_mutex_);
+
     if (encoder_thread_.joinable()) {
         encoder_thread_.join();
     }
@@ -394,6 +432,10 @@ bool EspVideo::Capture() {
         return false;
     }
 
+    if (!show_preview && esp_timer_get_time() < foreground_capture_protected_until_us_) {
+        return true;
+    }
+
     for (int i = 0; i < 3; i++) {
         struct v4l2_buffer buf = {};
         buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
@@ -729,9 +771,14 @@ bool EspVideo::Capture() {
         }
     }
 
-    // 显示预览图片
-    auto display = dynamic_cast<LvglDisplay*>(Board::GetInstance().GetDisplay());
-    if (display != nullptr) {
+    if (show_preview) {
+        foreground_capture_protected_until_us_ = esp_timer_get_time() + FOREGROUND_CAPTURE_PROTECTION_US;
+    }
+
+    if (show_preview) {
+        // 显示预览图片
+        auto display = dynamic_cast<LvglDisplay*>(Board::GetInstance().GetDisplay());
+        if (display != nullptr) {
         if (!frame_.data) {
             ESP_LOGE(TAG, "frame.data is null");
             return false;
@@ -836,6 +883,7 @@ bool EspVideo::Capture() {
 
         auto image = std::make_unique<LvglAllocatedImage>(data, lvgl_image_size, w, h, stride, color_format);
         display->SetPreviewImage(std::move(image));
+        }
     }
     return true;
 }
@@ -898,10 +946,16 @@ bool EspVideo::SetVFlip(bool enabled) {
  * @warning 如果摄像头缓冲区为空或网络连接失败，将返回错误信息
  */
 std::string EspVideo::Explain(const std::string& question) {
+    std::lock_guard<std::mutex> lock(frame_mutex_);
+
     if (explain_url_.empty()) {
         throw std::runtime_error("Image explain URL or token is not set");
     }
 
+    if (frame_.data == nullptr || frame_.len == 0) {
+        throw std::runtime_error("No camera frame captured");
+    }
+
     // 创建局部的 JPEG 队列, 40 entries is about to store 512 * 40 = 20480 bytes of JPEG data
     QueueHandle_t jpeg_queue = xQueueCreate(40, sizeof(JpegChunk));
     if (jpeg_queue == nullptr) {

main/boards/common/esp_video.h

@@ -5,6 +5,8 @@
 #include <thread>
 #include <memory>
 #include <vector>
+#include <mutex>
+#include <cstdint>
 
 #include <freertos/FreeRTOS.h>
 #include <freertos/queue.h>
@@ -39,6 +41,10 @@ private:
     std::string explain_url_;
     std::string explain_token_;
     std::thread encoder_thread_;
+    std::mutex frame_mutex_;
+    int64_t foreground_capture_protected_until_us_ = 0;
+
+    bool CaptureFrame(bool show_preview);
 
 public:
     EspVideo(const esp_video_init_config_t& config);
@@ -46,6 +52,8 @@ public:
 
     virtual void SetExplainUrl(const std::string& url, const std::string& token);
     virtual bool Capture();
+    virtual bool CaptureBackground() override;
+    virtual bool CaptureToJpeg(std::string& jpeg_data, bool show_preview = false) override;
     // 翻转控制函数
     virtual bool SetHMirror(bool enabled) override;
     virtual bool SetVFlip(bool enabled) override;