feat: support camera capture to livekit

main/application.cc

@@ -673,10 +673,17 @@ void Application::DismissAlert() {
 }
 
 void Application::ToggleChatState() {
+    vision_text_mode_enabled_.store(false);
+    xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT);
+}
+
+void Application::ToggleChatStateWithVision() {
+    vision_text_mode_enabled_.store(true);
     xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT);
 }
 
 void Application::StartListening() {
+    vision_text_mode_enabled_.store(false);
     xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING);
 }
 
@@ -686,7 +693,10 @@ void Application::StopListening() {
 
 void Application::HandleToggleChatEvent() {
     auto state = GetDeviceState();
-    
+    if (state != kDeviceStateIdle) {
+        vision_text_mode_enabled_.store(false);
+    }
+
     if (state == kDeviceStateActivating) {
         SetDeviceState(kDeviceStateIdle);
         return;
@@ -905,6 +915,9 @@ void Application::HandleStateChangedEvent() {
                     audio_service_.WaitForPlaybackQueueEmpty();
                 }
                 
+                if (vision_text_mode_enabled_.load()) {
+                    SendCurrentVisionFrame();
+                }
                 // Send the start listening command
                 protocol_->SendStartListening(listening_mode_);
                 audio_service_.EnableVoiceProcessing(true);
@@ -944,6 +957,26 @@ void Application::HandleStateChangedEvent() {
     }
 }
 
+void Application::SendCurrentVisionFrame() {
+    if (!protocol_ || !protocol_->IsAudioChannelOpened()) {
+        return;
+    }
+
+    auto camera = Board::GetInstance().GetCamera();
+    if (camera == nullptr) {
+        return;
+    }
+
+    std::string jpeg_data;
+    if (!camera->CaptureToJpeg(jpeg_data, false)) {
+        ESP_LOGW(TAG, "Failed to capture vision frame");
+        return;
+    }
+
+    protocol_->SendVisionFrame(jpeg_data);
+    ESP_LOGI(TAG, "Sent vision frame, size=%u bytes", static_cast<unsigned>(jpeg_data.size()));
+}
+
 void Application::Schedule(std::function<void()>&& callback) {
     {
         std::lock_guard<std::mutex> lock(mutex_);