feat: support camera capture to livekit

main/protocols/protocol.cc

@@ -1,9 +1,22 @@
 #include "protocol.h"
 
 #include <esp_log.h>
+#include <mbedtls/base64.h>
 
 #define TAG "Protocol"
 
+static std::string Base64Encode(const std::string& data) {
+    size_t encoded_length = 0;
+    size_t output_length = 0;
+    mbedtls_base64_encode(nullptr, 0, &encoded_length,
+                          reinterpret_cast<const unsigned char*>(data.data()), data.size());
+    std::string result(encoded_length, 0);
+    mbedtls_base64_encode(reinterpret_cast<unsigned char*>(result.data()), result.size(), &output_length,
+                          reinterpret_cast<const unsigned char*>(data.data()), data.size());
+    result.resize(output_length);
+    return result;
+}
+
 void Protocol::OnIncomingJson(std::function<void(const cJSON* root)> callback) {
     on_incoming_json_ = callback;
 }
@@ -78,6 +91,27 @@ void Protocol::SendMcpMessage(const std::string& payload) {
     SendText(message);
 }
 
+void Protocol::SendVisionFrame(const std::string& jpeg_data) {
+    if (jpeg_data.empty()) {
+        return;
+    }
+
+    cJSON* root = cJSON_CreateObject();
+    cJSON_AddStringToObject(root, "session_id", session_id_.c_str());
+    cJSON_AddStringToObject(root, "type", "vision");
+    cJSON_AddStringToObject(root, "state", "frame");
+    cJSON_AddStringToObject(root, "mime_type", "image/jpeg");
+    auto encoded = Base64Encode(jpeg_data);
+    cJSON_AddStringToObject(root, "image", encoded.c_str());
+
+    char* json_str = cJSON_PrintUnformatted(root);
+    if (json_str != nullptr) {
+        SendText(json_str);
+        cJSON_free(json_str);
+    }
+    cJSON_Delete(root);
+}
+
 bool Protocol::IsTimeout() const {
     const int kTimeoutSeconds = 120;
     auto now = std::chrono::steady_clock::now();

main/protocols/protocol.h

@@ -73,6 +73,7 @@ public:
     virtual void SendStopListening();
     virtual void SendAbortSpeaking(AbortReason reason);
     virtual void SendMcpMessage(const std::string& message);
+    virtual void SendVisionFrame(const std::string& jpeg_data);
 
 protected:
     std::function<void(const cJSON* root)> on_incoming_json_;
@@ -95,4 +96,3 @@ protected:
 };
 
 #endif // PROTOCOL_H
-