From a8c6c62c92d94b9eb4f5cb7b66ecf394e3680702 Mon Sep 17 00:00:00 2001 From: 0Xiao0 <511201264@qq.com> Date: Wed, 17 Jun 2026 15:02:16 +0800 Subject: [PATCH] feat: cam and slience detect sleep --- main/Kconfig.projbuild | 30 ++ main/application.cc | 407 +++++++++++------- main/application.h | 3 + main/boards/common/camera.h | 8 + main/boards/common/esp32_camera.cc | 139 ++++++ main/boards/common/esp32_camera.h | 2 + .../zhengchen-cam/custom-emojis/README.md | 7 +- main/boards/zhengchen-cam/mcp_controller.cc | 123 +++++- main/display/lcd_display.cc | 8 + 9 files changed, 564 insertions(+), 163 deletions(-) diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index b1e9b64..1941ae6 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -141,6 +141,36 @@ config ZHENGCHEN_CAM_USE_GIF_EMOJI When enabled, default assets use custom-emojis/gif. When disabled, default assets use custom-emojis/png. +menu "Fatigue Reminder" + depends on BOARD_TYPE_ZHENGCHEN_CAM || BOARD_TYPE_ZHENGCHEN_CAM_ML307 + +choice FATIGUE_DETECTION_DEFAULT_MODE + prompt "Default fatigue detection mode" + default FATIGUE_DETECTION_MODE_CAMERA + help + Select the default fatigue reminder detection mode used when no runtime NVS + override exists. + + config FATIGUE_DETECTION_MODE_CAMERA + bool "Camera blink/eye closure only" + config FATIGUE_DETECTION_MODE_SILENCE + bool "Silence/no-speaking only" + config FATIGUE_DETECTION_MODE_BOTH + bool "Camera and silence" + config FATIGUE_DETECTION_MODE_OFF + bool "Off" +endchoice + +config FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT + bool "Show camera debug preview by default" + depends on FATIGUE_DETECTION_MODE_CAMERA || FATIGUE_DETECTION_MODE_BOTH + default n + help + Show sampled camera frames while running local drowsiness detection. + This is useful for tuning, but it can cover the neutral standby image. + +endmenu + choice depends on BOARD_TYPE_LILYGO_T_DISPLAY_P4 prompt "Select the screen type" diff --git a/main/application.cc b/main/application.cc index 7b65794..6c8a010 100644 --- a/main/application.cc +++ b/main/application.cc @@ -1,24 +1,25 @@ #include "application.h" +#include "assets.h" +#include "assets/lang_config.h" +#include "audio_codec.h" #include "board.h" #include "display.h" -#include "system_info.h" -#include "audio_codec.h" -#include "mqtt_protocol.h" -#include "websocket_protocol.h" -#include "assets/lang_config.h" #include "mcp_server.h" -#include "assets.h" +#include "mqtt_protocol.h" #include "settings.h" +#include "system_info.h" +#include "websocket_protocol.h" -#include -#include -#include #include +#include +#include #include +#include #include #include #include #include +#include #define TAG "Application" @@ -28,10 +29,34 @@ constexpr int kDirectWebsocketVersion = 3; constexpr bool kUseDirectWebsocketWithoutOta = true; constexpr int kDefaultFatigueListeningTimeoutSec = 12; constexpr int kDefaultFatigueIdleTimeoutSec = 12; +constexpr int kDefaultFatigueCameraIntervalSec = 3; +constexpr int kDefaultFatigueCameraClosedSamples = 2; constexpr int kDefaultFatigueCooldownSec = 60; constexpr const char* kDefaultFatigueEmotion = "wakeup"; constexpr const char* kDefaultFatigueMessage = "你是不是又要睡着啦？快醒醒，我还要给你跳舞呢～"; constexpr const char* kDefaultFatigueStatus = "打起精神"; +#if defined(CONFIG_FATIGUE_DETECTION_MODE_OFF) +constexpr bool kDefaultFatigueEnabled = false; +constexpr bool kDefaultFatigueCameraEnabled = false; +constexpr bool kDefaultFatigueSilenceEnabled = false; +#elif defined(CONFIG_FATIGUE_DETECTION_MODE_SILENCE) +constexpr bool kDefaultFatigueEnabled = true; +constexpr bool kDefaultFatigueCameraEnabled = false; +constexpr bool kDefaultFatigueSilenceEnabled = true; +#elif defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH) +constexpr bool kDefaultFatigueEnabled = true; +constexpr bool kDefaultFatigueCameraEnabled = true; +constexpr bool kDefaultFatigueSilenceEnabled = true; +#else +constexpr bool kDefaultFatigueEnabled = true; +constexpr bool kDefaultFatigueCameraEnabled = true; +constexpr bool kDefaultFatigueSilenceEnabled = false; +#endif +#if defined(CONFIG_FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT) +constexpr bool kDefaultFatigueCameraDebugPreview = true; +#else +constexpr bool kDefaultFatigueCameraDebugPreview = false; +#endif void StartDirectTimeSync() { setenv("TZ", "CST-8", 1); @@ -57,8 +82,7 @@ void ConfigureDirectWebsocket() { settings.SetInt("version", kDirectWebsocketVersion); } } -} // namespace - +} // namespace Application::Application() { event_group_ = xEventGroupCreate(); @@ -73,16 +97,16 @@ Application::Application() { aec_mode_ = kAecOff; #endif - esp_timer_create_args_t clock_timer_args = { - .callback = [](void* arg) { - Application* app = (Application*)arg; - xEventGroupSetBits(app->event_group_, MAIN_EVENT_CLOCK_TICK); - }, - .arg = this, - .dispatch_method = ESP_TIMER_TASK, - .name = "clock_timer", - .skip_unhandled_events = true - }; + esp_timer_create_args_t clock_timer_args = {.callback = + [](void* arg) { + Application* app = (Application*)arg; + xEventGroupSetBits(app->event_group_, + MAIN_EVENT_CLOCK_TICK); + }, + .arg = this, + .dispatch_method = ESP_TIMER_TASK, + .name = "clock_timer", + .skip_unhandled_events = true}; esp_timer_create(&clock_timer_args, &clock_timer_handle_); } @@ -94,9 +118,7 @@ Application::~Application() { vEventGroupDelete(event_group_); } -bool Application::SetDeviceState(DeviceState state) { - return state_machine_.TransitionTo(state); -} +bool Application::SetDeviceState(DeviceState state) { return state_machine_.TransitionTo(state); } void Application::Initialize() { auto& board = Board::GetInstance(); @@ -141,7 +163,7 @@ void Application::Initialize() { // Set network event callback for UI updates and network state handling board.SetNetworkEventCallback([this](NetworkEvent event, const std::string& data) { auto display = Board::GetInstance().GetDisplay(); - + switch (event) { case NetworkEvent::Scanning: display->ShowNotification(Lang::Strings::SCANNING_WIFI, 30000); @@ -181,13 +203,16 @@ void Application::Initialize() { display->SetStatus(Lang::Strings::DETECTING_MODULE); break; case NetworkEvent::ModemErrorNoSim: - Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_PIN); + Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation", + Lang::Sounds::OGG_ERR_PIN); break; case NetworkEvent::ModemErrorRegDenied: - Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_REG); + Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation", + Lang::Sounds::OGG_ERR_REG); break; case NetworkEvent::ModemErrorInitFailed: - Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation", Lang::Sounds::OGG_EXCLAMATION); + Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation", + Lang::Sounds::OGG_EXCLAMATION); break; case NetworkEvent::ModemErrorTimeout: display->SetStatus(Lang::Strings::REGISTERING_NETWORK); @@ -206,19 +231,11 @@ void Application::Run() { // Set the priority of the main task to 10 vTaskPrioritySet(nullptr, 10); - const EventBits_t ALL_EVENTS = - MAIN_EVENT_SCHEDULE | - MAIN_EVENT_SEND_AUDIO | - MAIN_EVENT_WAKE_WORD_DETECTED | - MAIN_EVENT_VAD_CHANGE | - MAIN_EVENT_CLOCK_TICK | - MAIN_EVENT_ERROR | - MAIN_EVENT_NETWORK_CONNECTED | - MAIN_EVENT_NETWORK_DISCONNECTED | - MAIN_EVENT_TOGGLE_CHAT | - MAIN_EVENT_START_LISTENING | - MAIN_EVENT_STOP_LISTENING | - MAIN_EVENT_ACTIVATION_DONE | + const EventBits_t ALL_EVENTS = + MAIN_EVENT_SCHEDULE | MAIN_EVENT_SEND_AUDIO | MAIN_EVENT_WAKE_WORD_DETECTED | + MAIN_EVENT_VAD_CHANGE | MAIN_EVENT_CLOCK_TICK | MAIN_EVENT_ERROR | + MAIN_EVENT_NETWORK_CONNECTED | MAIN_EVENT_NETWORK_DISCONNECTED | MAIN_EVENT_TOGGLE_CHAT | + MAIN_EVENT_START_LISTENING | MAIN_EVENT_STOP_LISTENING | MAIN_EVENT_ACTIVATION_DONE | MAIN_EVENT_STATE_CHANGED; while (true) { @@ -226,7 +243,8 @@ void Application::Run() { if (bits & MAIN_EVENT_ERROR) { SetDeviceState(kDeviceStateIdle); - Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); + Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark", + Lang::Sounds::OGG_EXCLAMATION); } if (bits & MAIN_EVENT_NETWORK_CONNECTED) { @@ -290,7 +308,7 @@ void Application::Run() { auto display = Board::GetInstance().GetDisplay(); display->UpdateStatusBar(); CheckFatigueReminder(); - + // Print debug info every 10 seconds if (clock_ticks_ % 10 == 0) { SystemInfo::PrintHeapStats(); @@ -311,12 +329,14 @@ void Application::HandleNetworkConnectedEvent() { return; } - xTaskCreate([](void* arg) { - Application* app = static_cast(arg); - app->ActivationTask(); - app->activation_task_handle_ = nullptr; - vTaskDelete(NULL); - }, "activation", 4096 * 2, this, 2, &activation_task_handle_); + xTaskCreate( + [](void* arg) { + Application* app = static_cast(arg); + app->ActivationTask(); + app->activation_task_handle_ = nullptr; + vTaskDelete(NULL); + }, + "activation", 4096 * 2, this, 2, &activation_task_handle_); } // Update the status bar immediately to show the network state @@ -327,7 +347,8 @@ void Application::HandleNetworkConnectedEvent() { void Application::HandleNetworkDisconnectedEvent() { // Close current conversation when network disconnected auto state = GetDeviceState(); - if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) { + if (state == kDeviceStateConnecting || state == kDeviceStateListening || + state == kDeviceStateSpeaking) { ESP_LOGI(TAG, "Closing audio channel due to network disconnection"); protocol_->CloseAudioChannel(); } @@ -403,7 +424,7 @@ void Application::CheckAssetsVersion() { ESP_LOGW(TAG, "Assets partition is disabled for board %s", BOARD_NAME); return; } - + Settings settings("assets", true); // Check if there is a new assets need to be downloaded std::string download_url = settings.GetString("download_url"); @@ -413,27 +434,30 @@ void Application::CheckAssetsVersion() { char message[256]; snprintf(message, sizeof(message), Lang::Strings::FOUND_NEW_ASSETS, download_url.c_str()); - Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down", Lang::Sounds::OGG_UPGRADE); - + Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down", + Lang::Sounds::OGG_UPGRADE); + // Wait for the audio service to be idle for 3 seconds vTaskDelay(pdMS_TO_TICKS(3000)); SetDeviceState(kDeviceStateUpgrading); board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); display->SetChatMessage("system", Lang::Strings::PLEASE_WAIT); - bool success = assets.Download(download_url, [this, display](int progress, size_t speed) -> void { - char buffer[32]; - snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024); - Schedule([display, message = std::string(buffer)]() { - display->SetChatMessage("system", message.c_str()); + bool success = + assets.Download(download_url, [this, display](int progress, size_t speed) -> void { + char buffer[32]; + snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024); + Schedule([display, message = std::string(buffer)]() { + display->SetChatMessage("system", message.c_str()); + }); }); - }); board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); vTaskDelay(pdMS_TO_TICKS(1000)); if (!success) { - Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); + Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark", + Lang::Sounds::OGG_EXCLAMATION); vTaskDelay(pdMS_TO_TICKS(2000)); SetDeviceState(kDeviceStateActivating); return; @@ -449,7 +473,7 @@ void Application::CheckAssetsVersion() { void Application::CheckNewVersion() { const int MAX_RETRY = 10; int retry_count = 0; - int retry_delay = 10; // Initial retry delay in seconds + int retry_delay = 10; // Initial retry delay in seconds auto& board = Board::GetInstance(); while (true) { @@ -465,27 +489,30 @@ void Application::CheckNewVersion() { } char error_message[128]; - snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err, ota_->GetCheckVersionUrl().c_str()); + snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err, + ota_->GetCheckVersionUrl().c_str()); char buffer[256]; - snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay, error_message); + snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay, + error_message); Alert(Lang::Strings::ERROR, buffer, "cloud_slash", Lang::Sounds::OGG_EXCLAMATION); - ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay, retry_count, MAX_RETRY); + ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay, + retry_count, MAX_RETRY); for (int i = 0; i < retry_delay; i++) { vTaskDelay(pdMS_TO_TICKS(1000)); if (GetDeviceState() == kDeviceStateIdle) { break; } } - retry_delay *= 2; // Double the retry delay + retry_delay *= 2; // Double the retry delay continue; } retry_count = 0; - retry_delay = 10; // Reset retry delay + retry_delay = 10; // Reset retry delay if (ota_->HasNewVersion()) { if (UpgradeFirmware(ota_->GetFirmwareUrl(), ota_->GetFirmwareVersion())) { - return; // This line will never be reached after reboot + return; // This line will never be reached after reboot } // If upgrade failed, continue to normal operation } @@ -540,29 +567,29 @@ void Application::InitializeProtocol() { protocol_ = std::make_unique(); } - protocol_->OnConnected([this]() { - DismissAlert(); - }); + protocol_->OnConnected([this]() { DismissAlert(); }); protocol_->OnNetworkError([this](const std::string& message) { last_error_message_ = message; xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR); }); - + protocol_->OnIncomingAudio([this](std::unique_ptr packet) { if (GetDeviceState() == kDeviceStateSpeaking) { audio_service_.PushPacketToDecodeQueue(std::move(packet)); } }); - + protocol_->OnAudioChannelOpened([this, codec, &board]() { board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); if (protocol_->server_sample_rate() != codec->output_sample_rate()) { - ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion", - protocol_->server_sample_rate(), codec->output_sample_rate()); + ESP_LOGW(TAG, + "Server sample rate %d does not match device output sample rate %d, " + "resampling may cause distortion", + protocol_->server_sample_rate(), codec->output_sample_rate()); } }); - + protocol_->OnAudioChannelClosed([this, &board]() { board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); Schedule([this]() { @@ -571,7 +598,7 @@ void Application::InitializeProtocol() { SetDeviceState(kDeviceStateIdle); }); }); - + protocol_->OnIncomingJson([this, display](const cJSON* root) { // Parse JSON data auto type = cJSON_GetObjectItem(root, "type"); @@ -627,9 +654,7 @@ void Application::InitializeProtocol() { ESP_LOGI(TAG, "System command: %s", command->valuestring); if (strcmp(command->valuestring, "reboot") == 0) { // Do a reboot if user requests a OTA update - Schedule([this]() { - Reboot(); - }); + Schedule([this]() { Reboot(); }); } else { ESP_LOGW(TAG, "Unknown system command: %s", command->valuestring); } @@ -639,7 +664,8 @@ void Application::InitializeProtocol() { auto message = cJSON_GetObjectItem(root, "message"); auto emotion = cJSON_GetObjectItem(root, "emotion"); if (cJSON_IsString(status) && cJSON_IsString(message) && cJSON_IsString(emotion)) { - Alert(status->valuestring, message->valuestring, emotion->valuestring, Lang::Sounds::OGG_VIBRATION); + Alert(status->valuestring, message->valuestring, emotion->valuestring, + Lang::Sounds::OGG_VIBRATION); } else { ESP_LOGW(TAG, "Alert command requires status, message and emotion"); } @@ -648,9 +674,10 @@ void Application::InitializeProtocol() { auto payload = cJSON_GetObjectItem(root, "payload"); ESP_LOGI(TAG, "Received custom message: %s", cJSON_PrintUnformatted(root)); if (cJSON_IsObject(payload)) { - Schedule([this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() { - display->SetChatMessage("system", payload_str.c_str()); - }); + Schedule( + [this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() { + display->SetChatMessage("system", payload_str.c_str()); + }); } else { ESP_LOGW(TAG, "Invalid custom message format: missing payload"); } @@ -659,7 +686,7 @@ void Application::InitializeProtocol() { ESP_LOGW(TAG, "Unknown message type: %s", type->valuestring); } }); - + protocol_->Start(); } @@ -668,32 +695,27 @@ void Application::ShowActivationCode(const std::string& code, const std::string& char digit; const std::string_view& sound; }; - static const std::array digit_sounds{{ - digit_sound{'0', Lang::Sounds::OGG_0}, - digit_sound{'1', Lang::Sounds::OGG_1}, - digit_sound{'2', Lang::Sounds::OGG_2}, - digit_sound{'3', Lang::Sounds::OGG_3}, - digit_sound{'4', Lang::Sounds::OGG_4}, - digit_sound{'5', Lang::Sounds::OGG_5}, - digit_sound{'6', Lang::Sounds::OGG_6}, - digit_sound{'7', Lang::Sounds::OGG_7}, - digit_sound{'8', Lang::Sounds::OGG_8}, - digit_sound{'9', Lang::Sounds::OGG_9} - }}; + static const std::array digit_sounds{ + {digit_sound{'0', Lang::Sounds::OGG_0}, digit_sound{'1', Lang::Sounds::OGG_1}, + digit_sound{'2', Lang::Sounds::OGG_2}, digit_sound{'3', Lang::Sounds::OGG_3}, + digit_sound{'4', Lang::Sounds::OGG_4}, digit_sound{'5', Lang::Sounds::OGG_5}, + digit_sound{'6', Lang::Sounds::OGG_6}, digit_sound{'7', Lang::Sounds::OGG_7}, + digit_sound{'8', Lang::Sounds::OGG_8}, digit_sound{'9', Lang::Sounds::OGG_9}}}; // This sentence uses 9KB of SRAM, so we need to wait for it to finish Alert(Lang::Strings::ACTIVATION, message.c_str(), "link", Lang::Sounds::OGG_ACTIVATION); for (const auto& digit : code) { auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(), - [digit](const digit_sound& ds) { return ds.digit == digit; }); + [digit](const digit_sound& ds) { return ds.digit == digit; }); if (it != digit_sounds.end()) { audio_service_.PlaySound(it->sound); } } } -void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) { +void Application::Alert(const char* status, const char* message, const char* emotion, + const std::string_view& sound) { ESP_LOGW(TAG, "Alert [%s] %s: %s", emotion, status, message); auto display = Board::GetInstance().GetDisplay(); display->SetStatus(status); @@ -724,7 +746,7 @@ void Application::CheckFatigueReminder() { } Settings settings("fatigue", false); - if (!settings.GetBool("enabled", true)) { + if (!settings.GetBool("enabled", kDefaultFatigueEnabled)) { return; } @@ -740,6 +762,19 @@ void Application::CheckFatigueReminder() { return; } + if (CheckCameraDrowsiness()) { + last_fatigue_reminder_time_us_ = esp_timer_get_time(); + TriggerFatigueReminder(); + return; + } + + if (!settings.GetBool("silence_enabled", kDefaultFatigueSilenceEnabled)) { + fatigue_idle_seconds_ = 0; + fatigue_silence_seconds_ = 0; + fatigue_reminder_triggered_in_listening_ = false; + return; + } + if (state == kDeviceStateIdle) { fatigue_idle_seconds_++; @@ -787,14 +822,79 @@ void Application::CheckFatigueReminder() { TriggerFatigueReminder(); } +bool Application::CheckCameraDrowsiness() { + auto state = GetDeviceState(); + if (state != kDeviceStateIdle && state != kDeviceStateListening) { + fatigue_camera_closed_samples_ = 0; + return false; + } + + Settings settings("fatigue", false); + if (!settings.GetBool("camera_enabled", kDefaultFatigueCameraEnabled)) { + fatigue_camera_closed_samples_ = 0; + return false; + } + + int interval_sec = settings.GetInt("camera_interval_sec", kDefaultFatigueCameraIntervalSec); + if (interval_sec < 1) { + interval_sec = 1; + } else if (interval_sec > 60) { + interval_sec = 60; + } + + int64_t now_us = esp_timer_get_time(); + if (last_fatigue_camera_check_time_us_ != 0 && + now_us - last_fatigue_camera_check_time_us_ < + static_cast(interval_sec) * 1000000) { + return false; + } + last_fatigue_camera_check_time_us_ = now_us; + + auto camera = Board::GetInstance().GetCamera(); + if (camera == nullptr) { + return false; + } + + bool show_debug_preview = + settings.GetBool("camera_debug_preview", kDefaultFatigueCameraDebugPreview); + CameraDrowsinessResult result; + if (!camera->DetectDrowsiness(result, show_debug_preview)) { + fatigue_camera_closed_samples_ = 0; + return false; + } + + ESP_LOGI(TAG, "Camera drowsiness: closed=%d score=%.2f baseline=%.2f", + result.eyes_closed ? 1 : 0, result.eye_openness_score, result.baseline_score); + + if (result.eyes_closed) { + fatigue_camera_closed_samples_++; + } else { + fatigue_camera_closed_samples_ = 0; + } + + int required_samples = + settings.GetInt("camera_closed_samples", kDefaultFatigueCameraClosedSamples); + if (required_samples < 1) { + required_samples = 1; + } else if (required_samples > 10) { + required_samples = 10; + } + + if (fatigue_camera_closed_samples_ >= required_samples) { + fatigue_camera_closed_samples_ = 0; + return true; + } + return false; +} + void Application::TriggerFatigueReminder() { Settings settings("fatigue", false); std::string emotion = settings.GetString("emotion", kDefaultFatigueEmotion); std::string message = settings.GetString("message", kDefaultFatigueMessage); std::string sound_asset = settings.GetString("sound_asset"); - ESP_LOGW(TAG, "Fatigue reminder triggered: silence=%ds, emotion=%s", - fatigue_silence_seconds_, emotion.c_str()); + ESP_LOGW(TAG, "Fatigue reminder triggered: silence=%ds, emotion=%s", fatigue_silence_seconds_, + emotion.c_str()); auto display = Board::GetInstance().GetDisplay(); display->SetStatus(kDefaultFatigueStatus); @@ -815,21 +915,15 @@ void Application::TriggerFatigueReminder() { audio_service_.PlaySound(Lang::Sounds::OGG_POPUP); } -void Application::ToggleChatState() { - xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT); -} +void Application::ToggleChatState() { xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT); } -void Application::StartListening() { - xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING); -} +void Application::StartListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING); } -void Application::StopListening() { - xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING); -} +void Application::StopListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING); } void Application::HandleToggleChatEvent() { auto state = GetDeviceState(); - + if (state == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); return; @@ -853,9 +947,7 @@ void Application::HandleToggleChatEvent() { if (!protocol_->IsAudioChannelOpened()) { SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update) - Schedule([this, mode]() { - ContinueOpenAudioChannel(mode); - }); + Schedule([this, mode]() { ContinueOpenAudioChannel(mode); }); return; } SetListeningMode(mode); @@ -883,7 +975,7 @@ void Application::ContinueOpenAudioChannel(ListeningMode mode) { void Application::HandleStartListeningEvent() { auto state = GetDeviceState(); - + if (state == kDeviceStateActivating) { SetDeviceState(kDeviceStateIdle); return; @@ -897,14 +989,12 @@ void Application::HandleStartListeningEvent() { ESP_LOGE(TAG, "Protocol not initialized"); return; } - + if (state == kDeviceStateIdle) { if (!protocol_->IsAudioChannelOpened()) { SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update) - Schedule([this]() { - ContinueOpenAudioChannel(kListeningModeManualStop); - }); + Schedule([this]() { ContinueOpenAudioChannel(kListeningModeManualStop); }); return; } SetListeningMode(kListeningModeManualStop); @@ -916,7 +1006,7 @@ void Application::HandleStartListeningEvent() { void Application::HandleStopListeningEvent() { auto state = GetDeviceState(); - + if (state == kDeviceStateAudioTesting) { audio_service_.EnableAudioTesting(false); SetDeviceState(kDeviceStateWifiConfiguring); @@ -946,9 +1036,7 @@ void Application::HandleWakeWordDetectedEvent() { SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update), // then continue with OpenAudioChannel which may block for ~1 second - Schedule([this, wake_word]() { - ContinueWakeWordInvoke(wake_word); - }); + Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); }); return; } // Channel already opened, continue directly @@ -956,7 +1044,8 @@ void Application::HandleWakeWordDetectedEvent() { } else if (state == kDeviceStateSpeaking || state == kDeviceStateListening) { AbortSpeaking(kAbortReasonWakeWordDetected); // Clear send queue to avoid sending residues to server - while (audio_service_.PopPacketFromSendQueue()); + while (audio_service_.PopPacketFromSendQueue()) + ; if (state == kDeviceStateListening) { protocol_->SendStartListening(GetDefaultListeningMode()); @@ -1016,13 +1105,13 @@ void Application::HandleStateChangedEvent() { auto display = board.GetDisplay(); auto led = board.GetLed(); led->OnStateChanged(); - + switch (new_state) { case kDeviceStateUnknown: case kDeviceStateIdle: display->SetStatus(Lang::Strings::STANDBY); - display->ClearChatMessages(); // Clear messages first - display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count) + display->ClearChatMessages(); // Clear messages first + display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count) audio_service_.EnableVoiceProcessing(false); audio_service_.EnableWakeWordDetection(true); break; @@ -1038,12 +1127,13 @@ void Application::HandleStateChangedEvent() { // Make sure the audio processor is running if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) { - // For auto mode, wait for playback queue to be empty before enabling voice processing - // This prevents audio truncation when STOP arrives late due to network jitter + // For auto mode, wait for playback queue to be empty before enabling voice + // processing This prevents audio truncation when STOP arrives late due to network + // jitter if (listening_mode_ == kListeningModeAutoStop) { audio_service_.WaitForPlaybackQueueEmpty(); } - + // Send the start listening command protocol_->SendStartListening(listening_mode_); audio_service_.EnableVoiceProcessing(true); @@ -1056,7 +1146,7 @@ void Application::HandleStateChangedEvent() { // Disable wake word detection in listening mode audio_service_.EnableWakeWordDetection(false); #endif - + // Play popup sound after ResetDecoder (in EnableVoiceProcessing) has been called if (play_popup_on_listening_) { play_popup_on_listening_ = false; @@ -1135,7 +1225,8 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver } ESP_LOGI(TAG, "Starting firmware upgrade from URL: %s", upgrade_url.c_str()); - Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download", Lang::Sounds::OGG_UPGRADE); + Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download", + Lang::Sounds::OGG_UPGRADE); vTaskDelay(pdMS_TO_TICKS(3000)); SetDeviceState(kDeviceStateUpgrading); @@ -1157,17 +1248,19 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver if (!upgrade_success) { // Upgrade failed, restart audio service and continue running - ESP_LOGE(TAG, "Firmware upgrade failed, restarting audio service and continuing operation..."); - audio_service_.Start(); // Restart audio service - board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level - Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); + ESP_LOGE(TAG, + "Firmware upgrade failed, restarting audio service and continuing operation..."); + audio_service_.Start(); // Restart audio service + board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level + Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark", + Lang::Sounds::OGG_EXCLAMATION); vTaskDelay(pdMS_TO_TICKS(3000)); return false; } else { // Upgrade success, reboot immediately ESP_LOGI(TAG, "Firmware upgrade successful, rebooting..."); display->SetChatMessage("system", "Upgrade successful, rebooting..."); - vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message + vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message Reboot(); return true; } @@ -1179,25 +1272,21 @@ void Application::WakeWordInvoke(const std::string& wake_word) { } auto state = GetDeviceState(); - + if (state == kDeviceStateIdle) { audio_service_.EncodeWakeWord(); if (!protocol_->IsAudioChannelOpened()) { SetDeviceState(kDeviceStateConnecting); // Schedule to let the state change be processed first (UI update) - Schedule([this, wake_word]() { - ContinueWakeWordInvoke(wake_word); - }); + Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); }); return; } // Channel already opened, continue directly ContinueWakeWordInvoke(wake_word); } else if (state == kDeviceStateSpeaking) { - Schedule([this]() { - AbortSpeaking(kAbortReasonNone); - }); - } else if (state == kDeviceStateListening) { + Schedule([this]() { AbortSpeaking(kAbortReasonNone); }); + } else if (state == kDeviceStateListening) { Schedule([this]() { if (protocol_) { protocol_->CloseAudioChannel(); @@ -1238,18 +1327,18 @@ void Application::SetAecMode(AecMode mode) { auto& board = Board::GetInstance(); auto display = board.GetDisplay(); switch (aec_mode_) { - case kAecOff: - audio_service_.EnableDeviceAec(false); - display->ShowNotification(Lang::Strings::RTC_MODE_OFF); - break; - case kAecOnServerSide: - audio_service_.EnableDeviceAec(false); - display->ShowNotification(Lang::Strings::RTC_MODE_ON); - break; - case kAecOnDeviceSide: - audio_service_.EnableDeviceAec(true); - display->ShowNotification(Lang::Strings::RTC_MODE_ON); - break; + case kAecOff: + audio_service_.EnableDeviceAec(false); + display->ShowNotification(Lang::Strings::RTC_MODE_OFF); + break; + case kAecOnServerSide: + audio_service_.EnableDeviceAec(false); + display->ShowNotification(Lang::Strings::RTC_MODE_ON); + break; + case kAecOnDeviceSide: + audio_service_.EnableDeviceAec(true); + display->ShowNotification(Lang::Strings::RTC_MODE_ON); + break; } // If the AEC mode is changed, close the audio channel @@ -1259,9 +1348,7 @@ void Application::SetAecMode(AecMode mode) { }); } -void Application::PlaySound(const std::string_view& sound) { - audio_service_.PlaySound(sound); -} +void Application::PlaySound(const std::string_view& sound) { audio_service_.PlaySound(sound); } void Application::ResetProtocol() { Schedule([this]() { diff --git a/main/application.h b/main/application.h index 1cea287..5dfac56 100644 --- a/main/application.h +++ b/main/application.h @@ -143,7 +143,9 @@ private: bool fatigue_reminder_triggered_in_listening_ = false; int fatigue_silence_seconds_ = 0; int fatigue_idle_seconds_ = 0; + int fatigue_camera_closed_samples_ = 0; int64_t last_fatigue_reminder_time_us_ = 0; + int64_t last_fatigue_camera_check_time_us_ = 0; int clock_ticks_ = 0; TaskHandle_t activation_task_handle_ = nullptr; @@ -160,6 +162,7 @@ private: void ContinueOpenAudioChannel(ListeningMode mode); void ContinueWakeWordInvoke(const std::string& wake_word); void CheckFatigueReminder(); + bool CheckCameraDrowsiness(); void TriggerFatigueReminder(); // Activation task (runs in background) diff --git a/main/boards/common/camera.h b/main/boards/common/camera.h index 09a6a3e..86947a2 100644 --- a/main/boards/common/camera.h +++ b/main/boards/common/camera.h @@ -3,6 +3,13 @@ #include +struct CameraDrowsinessResult { + bool valid = false; + bool eyes_closed = false; + float eye_openness_score = 0.0f; + float baseline_score = 0.0f; +}; + class Camera { public: virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0; @@ -10,6 +17,7 @@ public: virtual bool SetHMirror(bool enabled) = 0; virtual bool SetVFlip(bool enabled) = 0; virtual bool SetSwapBytes(bool enabled) { return false; } // Optional, default no-op + virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) { return false; } virtual std::string Explain(const std::string& question) = 0; }; diff --git a/main/boards/common/esp32_camera.cc b/main/boards/common/esp32_camera.cc index 76eaaab..2e243bb 100644 --- a/main/boards/common/esp32_camera.cc +++ b/main/boards/common/esp32_camera.cc @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "esp32_camera.h" #include "board.h" @@ -17,6 +19,38 @@ #define TAG "Esp32Camera" +namespace { +constexpr float kClosedEyeScoreRatio = 0.58f; +constexpr int kEyeSampleStep = 4; + +uint8_t Rgb565ToLuma(uint16_t pixel) { + uint8_t r = ((pixel >> 11) & 0x1f) << 3; + uint8_t g = ((pixel >> 5) & 0x3f) << 2; + uint8_t b = (pixel & 0x1f) << 3; + return static_cast((static_cast(r) * 30 + + static_cast(g) * 59 + + static_cast(b) * 11) / 100); +} + +void DrawRectRgb565(uint16_t* pixels, int width, int height, int x0, int y0, int x1, int y1, uint16_t color) { + if (pixels == nullptr || width <= 0 || height <= 0) { + return; + } + x0 = std::clamp(x0, 0, width - 1); + x1 = std::clamp(x1, 0, width - 1); + y0 = std::clamp(y0, 0, height - 1); + y1 = std::clamp(y1, 0, height - 1); + for (int x = x0; x <= x1; x++) { + pixels[y0 * width + x] = color; + pixels[y1 * width + x] = color; + } + for (int y = y0; y <= y1; y++) { + pixels[y * width + x0] = color; + pixels[y * width + x1] = color; + } +} +} // namespace + Esp32Camera::Esp32Camera(const camera_config_t &config) { esp_err_t err = esp_camera_init(&config); if (err != ESP_OK) { @@ -152,6 +186,111 @@ bool Esp32Camera::SetSwapBytes(bool enabled) { return true; } +bool Esp32Camera::DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview) { + result = {}; + + if (encoder_thread_.joinable()) { + encoder_thread_.join(); + } + if (!streaming_on_) { + return false; + } + + if (current_fb_) { + esp_camera_fb_return(current_fb_); + current_fb_ = nullptr; + } + + camera_fb_t* fb = esp_camera_fb_get(); + if (!fb) { + ESP_LOGW(TAG, "Camera drowsiness capture failed"); + return false; + } + + if (fb->format != PIXFORMAT_RGB565 || fb->width < 80 || fb->height < 80) { + ESP_LOGW(TAG, "Unsupported drowsiness frame: %dx%d format=%d", + fb->width, fb->height, fb->format); + esp_camera_fb_return(fb); + return false; + } + + const int width = fb->width; + const int height = fb->height; + const int x0 = width * 22 / 100; + const int x1 = width * 78 / 100; + const int y0 = height * 24 / 100; + const int y1 = height * 46 / 100; + const uint16_t* pixels = reinterpret_cast(fb->buf); + + if (show_debug_preview) { + size_t data_size = static_cast(width) * static_cast(height) * 2; + auto* preview_data = static_cast(heap_caps_malloc(data_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (preview_data != nullptr) { + auto* dst = reinterpret_cast(preview_data); + const auto* src = reinterpret_cast(fb->buf); + size_t pixel_count = static_cast(width) * static_cast(height); + for (size_t i = 0; i < pixel_count; i++) { + dst[i] = swap_bytes_enabled_ ? __builtin_bswap16(src[i]) : src[i]; + } + DrawRectRgb565(dst, width, height, x0, y0, x1, y1, 0xF800); + auto display = dynamic_cast(Board::GetInstance().GetDisplay()); + if (display != nullptr) { + display->SetPreviewImage(std::make_unique( + preview_data, data_size, width, height, width * 2, LV_COLOR_FORMAT_RGB565)); + } else { + heap_caps_free(preview_data); + } + } + } + + float vertical_edge_sum = 0.0f; + float horizontal_edge_sum = 0.0f; + int samples = 0; + for (int y = y0; y + kEyeSampleStep < y1; y += kEyeSampleStep) { + for (int x = x0; x + kEyeSampleStep < x1; x += kEyeSampleStep) { + uint16_t p = pixels[y * width + x]; + uint16_t px = pixels[y * width + x + kEyeSampleStep]; + uint16_t py = pixels[(y + kEyeSampleStep) * width + x]; + if (swap_bytes_enabled_) { + p = __builtin_bswap16(p); + px = __builtin_bswap16(px); + py = __builtin_bswap16(py); + } + uint8_t l = Rgb565ToLuma(p); + vertical_edge_sum += std::abs(static_cast(l) - static_cast(Rgb565ToLuma(py))); + horizontal_edge_sum += std::abs(static_cast(l) - static_cast(Rgb565ToLuma(px))); + samples++; + } + } + + esp_camera_fb_return(fb); + + if (samples == 0) { + return false; + } + + // Open eyes usually keep more vertical texture in the fixed eye band. + // This is a lightweight central-face heuristic, not a landmark model. + float score = (vertical_edge_sum + horizontal_edge_sum * 0.35f) / samples; + if (eye_openness_baseline_ <= 0.0f) { + eye_openness_baseline_ = score; + } else if (score > eye_openness_baseline_ * 0.85f) { + eye_openness_baseline_ = eye_openness_baseline_ * 0.90f + score * 0.10f; + } else { + eye_openness_baseline_ = eye_openness_baseline_ * 0.995f + score * 0.005f; + } + + result.valid = eye_openness_baseline_ > 1.0f; + result.eye_openness_score = score; + result.baseline_score = eye_openness_baseline_; + result.eyes_closed = result.valid && score < eye_openness_baseline_ * kClosedEyeScoreRatio; + + ESP_LOGI(TAG, "Drowsiness frame=%dx%d eye_roi=(%d,%d)-(%d,%d) score=%.2f baseline=%.2f closed=%d", + width, height, x0, y0, x1, y1, result.eye_openness_score, result.baseline_score, + result.eyes_closed ? 1 : 0); + return result.valid; +} + std::string Esp32Camera::Explain(const std::string &question) { if (explain_url_.empty()) { throw std::runtime_error("Image explain URL or token is not set"); diff --git a/main/boards/common/esp32_camera.h b/main/boards/common/esp32_camera.h index e2fb071..4e5a70f 100644 --- a/main/boards/common/esp32_camera.h +++ b/main/boards/common/esp32_camera.h @@ -30,6 +30,7 @@ private: camera_fb_t *current_fb_ = nullptr; uint8_t *encode_buf_ = nullptr; // Buffer for JPEG encoding (with optional byte swap) size_t encode_buf_size_ = 0; + float eye_openness_baseline_ = 0.0f; public: Esp32Camera(const camera_config_t &config); @@ -40,5 +41,6 @@ public: virtual bool SetHMirror(bool enabled) override; virtual bool SetVFlip(bool enabled) override; virtual bool SetSwapBytes(bool enabled) override; + virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) override; virtual std::string Explain(const std::string &question) override; }; diff --git a/main/boards/zhengchen-cam/custom-emojis/README.md b/main/boards/zhengchen-cam/custom-emojis/README.md index d9e1c13..b199656 100644 --- a/main/boards/zhengchen-cam/custom-emojis/README.md +++ b/main/boards/zhengchen-cam/custom-emojis/README.md @@ -29,9 +29,14 @@ Recommended minimum set: Fatigue reminder: - Add `wakeup.gif` or `wakeup.png` to make the idle-fatigue reminder show a custom idol animation. -- The reminder defaults to `wakeup` after 12 seconds of idle time or listening silence, then waits 60 seconds before it can trigger again. +- The reminder defaults to `wakeup` when camera drowsiness is detected, then waits 60 seconds before it can trigger again. - Optional NVS settings in namespace `fatigue`: - `enabled` (`bool`, default `true`) + - `camera_enabled` (`bool`, default `true`) + - `camera_debug_preview` (`bool`, default `true`; shows the sampled frame with the eye ROI box) + - `camera_interval_sec` (`int`, default `3`) + - `camera_closed_samples` (`int`, default `2`) + - `silence_enabled` (`bool`, default `false`) - `idle_timeout_sec` (`int`, default `12`) - `listening_timeout_sec` (`int`, default `12`) - `cooldown_sec` (`int`, default `60`) diff --git a/main/boards/zhengchen-cam/mcp_controller.cc b/main/boards/zhengchen-cam/mcp_controller.cc index 0f7f073..415f266 100644 --- a/main/boards/zhengchen-cam/mcp_controller.cc +++ b/main/boards/zhengchen-cam/mcp_controller.cc @@ -1,5 +1,6 @@ #include #include +#include #include @@ -13,6 +14,52 @@ #define TAG "MCPController" +namespace { +constexpr bool kDefaultFatigueEnabled = +#if defined(CONFIG_FATIGUE_DETECTION_MODE_OFF) + false; +#else + true; +#endif + +constexpr bool kDefaultFatigueCameraEnabled = +#if defined(CONFIG_FATIGUE_DETECTION_MODE_CAMERA) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH) + true; +#else + false; +#endif + +constexpr bool kDefaultFatigueSilenceEnabled = +#if defined(CONFIG_FATIGUE_DETECTION_MODE_SILENCE) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH) + true; +#else + false; +#endif + +constexpr bool kDefaultFatigueCameraDebugPreview = +#if defined(CONFIG_FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT) + true; +#else + false; +#endif + +std::string GetFatigueDetectionMode(bool enabled, bool camera_enabled, bool silence_enabled) { + if (!enabled) { + return "off"; + } + if (camera_enabled && silence_enabled) { + return "both"; + } + if (camera_enabled) { + return "camera"; + } + if (silence_enabled) { + return "silence"; + } + return "off"; +} +} // namespace + class MCPController { public: MCPController() { @@ -42,7 +89,6 @@ public: app.SetAecMode(kAecOff); return "{\"success\": true, \"message\": \"AEC对话打断模式已关闭\"}"; }else { - auto& board = Board::GetInstance(); app.SetAecMode(kAecOnDeviceSide); return "{\"success\": true, \"message\": \"AEC对话打断模式已开启\"}"; @@ -79,6 +125,79 @@ public: } ); + mcp_server.AddTool( + "self.fatigue.set_detection_mode", + "切换疲劳提醒检测模式。当用户想切换打瞌睡检测、眨眼检测、不讲话检测、关闭疲劳提醒或查看摄像头取样画面时使用此工具。\n" + "参数：\n" + " `mode`: 检测模式，可选值只有 `camera`(只用摄像头眨眼/闭眼检测)、`silence`(只用不讲话/静默检测)、`both`(两个都开)、`off`(关闭疲劳提醒)\n" + " `debug_preview`: 是否显示摄像头调试预览。默认 false；只有用户明确想看摄像头拍到什么时才设为 true\n" + "返回值：\n" + " 反馈当前疲劳检测模式，不需要确认，立即播报相关数据\n", + PropertyList({ + Property("mode", kPropertyTypeString), + Property("debug_preview", kPropertyTypeBoolean, false), + }), + [](const PropertyList& properties) -> ReturnValue { + auto mode = properties["mode"].value(); + bool debug_preview = properties["debug_preview"].value(); + + bool enabled = true; + bool camera_enabled = false; + bool silence_enabled = false; + std::string message; + + if (mode == "camera") { + camera_enabled = true; + message = "已切换为摄像头眨眼检测模式"; + } else if (mode == "silence") { + silence_enabled = true; + message = "已切换为不讲话静默检测模式"; + } else if (mode == "both") { + camera_enabled = true; + silence_enabled = true; + message = "已切换为摄像头加静默双检测模式"; + } else if (mode == "off") { + enabled = false; + message = "疲劳提醒检测已关闭"; + } else { + return "{\"success\": false, \"message\": \"检测模式无效，只能使用 camera、silence、both 或 off\"}"; + } + + Settings settings("fatigue", true); + settings.SetBool("enabled", enabled); + settings.SetBool("camera_enabled", camera_enabled); + settings.SetBool("silence_enabled", silence_enabled); + settings.SetBool("camera_debug_preview", debug_preview); + + return std::string("{\"success\": true, \"mode\": \"") + mode + + "\", \"debug_preview\": " + (debug_preview ? "true" : "false") + + ", \"message\": \"" + message + "\"}"; + } + ); + + mcp_server.AddTool( + "self.fatigue.get_detection_mode", + "获取当前疲劳提醒检测模式。当用户询问现在使用哪种打瞌睡检测模式时使用此工具。\n" + "返回值：\n" + " 当前检测模式和摄像头预览状态\n", + PropertyList(), + [](const PropertyList&) -> ReturnValue { + Settings settings("fatigue", false); + bool enabled = settings.GetBool("enabled", kDefaultFatigueEnabled); + bool camera_enabled = settings.GetBool("camera_enabled", kDefaultFatigueCameraEnabled); + bool silence_enabled = + settings.GetBool("silence_enabled", kDefaultFatigueSilenceEnabled); + bool debug_preview = + settings.GetBool("camera_debug_preview", kDefaultFatigueCameraDebugPreview); + auto mode = GetFatigueDetectionMode(enabled, camera_enabled, silence_enabled); + + return std::string("{\"success\": true, \"mode\": \"") + mode + + "\", \"camera_enabled\": " + (camera_enabled ? "true" : "false") + + ", \"silence_enabled\": " + (silence_enabled ? "true" : "false") + + ", \"debug_preview\": " + (debug_preview ? "true" : "false") + "}"; + } + ); + ESP_LOGI(TAG, "MCP工具注册完成"); } @@ -91,4 +210,4 @@ void InitializeMCPController() { g_mcp_controller = new MCPController(); ESP_LOGI(TAG, "注册MCP工具"); } -} \ No newline at end of file +} diff --git a/main/display/lcd_display.cc b/main/display/lcd_display.cc index 1c57ad9..943d436 100644 --- a/main/display/lcd_display.cc +++ b/main/display/lcd_display.cc @@ -1206,6 +1206,14 @@ void LcdDisplay::SetEmotion(const char* emotion) { DisplayLockGuard lock(this); bool use_full_screen_center = strcmp(emotion, "neutral") == 0; + if (use_full_screen_center && preview_image_ != nullptr) { + esp_timer_stop(preview_timer_); + if (emoji_box_ != nullptr) { + lv_obj_remove_flag(emoji_box_, LV_OBJ_FLAG_HIDDEN); + } + lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN); + preview_image_cached_.reset(); + } lv_coord_t top_reserved = use_full_screen_center ? 0 : ObjectHeight(top_bar_); lv_coord_t bottom_reserved = use_full_screen_center ? 0 : ObjectHeight(status_bar_) + ObjectHeight(bottom_bar_); auto apply_emoji_layout = [this, top_reserved, bottom_reserved](const lv_image_dsc_t* image_dsc) {