feat: cam and slience detect sleep

main/CMakeLists.txt

@@ -651,14 +651,14 @@ elseif(CONFIG_BOARD_TYPE_ZHENGCHEN_1_54TFT_ML307)
     set(DEFAULT_EMOJI_COLLECTION twemoji_64)
 elseif(CONFIG_BOARD_TYPE_ZHENGCHEN_CAM)
     set(BOARD_TYPE "zhengchen-cam")
-    set(BUILTIN_TEXT_FONT font_puhui_basic_20_4)
+    set(BUILTIN_TEXT_FONT font_puhui_20_4)
     set(BUILTIN_ICON_FONT font_awesome_20_4)
-    set(DEFAULT_EMOJI_COLLECTION twemoji_64)
+    set(DEFAULT_EMOJI_COLLECTION main/boards/zhengchen-cam/custom-emojis)
 elseif(CONFIG_BOARD_TYPE_ZHENGCHEN_CAM_ML307)
     set(BOARD_TYPE "zhengchen-cam-ml307")
-    set(BUILTIN_TEXT_FONT font_puhui_basic_20_4)
+    set(BUILTIN_TEXT_FONT font_puhui_20_4)
     set(BUILTIN_ICON_FONT font_awesome_20_4)
-    set(DEFAULT_EMOJI_COLLECTION twemoji_64)    
+    set(DEFAULT_EMOJI_COLLECTION main/boards/zhengchen-cam/custom-emojis)
 elseif(CONFIG_BOARD_TYPE_SPOTPEAR_ESP32_S3_1_54_MUMA)
     set(BOARD_TYPE "sp-esp32-s3-1.54-muma")
     set(BUILTIN_TEXT_FONT font_puhui_basic_16_4)
@@ -912,6 +912,7 @@ idf_component_register(SRCS ${SOURCES}
                         efuse
                         bt
                         fatfs
+                        lwip
                     )
 
 # Use target_compile_definitions to define BOARD_TYPE, BOARD_NAME
@@ -1017,6 +1018,25 @@ function(build_default_assets_bin)
         list(APPEND BUILD_ARGS "--extra_files" "${DEFAULT_ASSETS_EXTRA_FILES}")
     endif()
 
+    set(DEFAULT_ASSETS_DEPENDS
+        ${SDKCONFIG}
+        ${PROJECT_DIR}/scripts/build_default_assets.py
+    )
+    if(DEFAULT_EMOJI_COLLECTION)
+        if(IS_ABSOLUTE "${DEFAULT_EMOJI_COLLECTION}")
+            set(DEFAULT_EMOJI_COLLECTION_PATH "${DEFAULT_EMOJI_COLLECTION}")
+        else()
+            set(DEFAULT_EMOJI_COLLECTION_PATH "${PROJECT_DIR}/${DEFAULT_EMOJI_COLLECTION}")
+        endif()
+        if(IS_DIRECTORY "${DEFAULT_EMOJI_COLLECTION_PATH}")
+            file(GLOB_RECURSE DEFAULT_EMOJI_COLLECTION_FILES CONFIGURE_DEPENDS
+                "${DEFAULT_EMOJI_COLLECTION_PATH}/*.png"
+                "${DEFAULT_EMOJI_COLLECTION_PATH}/*.gif"
+            )
+            list(APPEND DEFAULT_ASSETS_DEPENDS ${DEFAULT_EMOJI_COLLECTION_FILES})
+        endif()
+    endif()
+    
     list(APPEND BUILD_ARGS "--esp_sr_model_path" "${ESP_SR_MODEL_PATH}")
     list(APPEND BUILD_ARGS "--xiaozhi_fonts_path" "${XIAOZHI_FONTS_PATH}")
     
@@ -1024,9 +1044,7 @@ function(build_default_assets_bin)
     add_custom_command(
         OUTPUT ${GENERATED_ASSETS_BIN}
         COMMAND python ${PROJECT_DIR}/scripts/build_default_assets.py ${BUILD_ARGS}
-        DEPENDS
-            ${SDKCONFIG}
-            ${PROJECT_DIR}/scripts/build_default_assets.py
+        DEPENDS ${DEFAULT_ASSETS_DEPENDS}
         COMMENT "Building default assets.bin based on configuration"
         VERBATIM
     )

main/Kconfig.projbuild

@@ -133,6 +133,44 @@ choice BOARD_TYPE
         depends on IDF_TARGET_ESP32S3
 endchoice
 
+config ZHENGCHEN_CAM_USE_GIF_EMOJI
+    bool "Use GIF custom emojis for Zhengchen CAM"
+    depends on BOARD_TYPE_ZHENGCHEN_CAM || BOARD_TYPE_ZHENGCHEN_CAM_ML307
+    default n
+    help
+        When enabled, default assets use custom-emojis/gif.
+        When disabled, default assets use custom-emojis/png.
+
+menu "Fatigue Reminder"
+    depends on BOARD_TYPE_ZHENGCHEN_CAM || BOARD_TYPE_ZHENGCHEN_CAM_ML307
+
+choice FATIGUE_DETECTION_DEFAULT_MODE
+    prompt "Default fatigue detection mode"
+    default FATIGUE_DETECTION_MODE_CAMERA
+    help
+        Select the default fatigue reminder detection mode used when no runtime NVS
+        override exists.
+
+    config FATIGUE_DETECTION_MODE_CAMERA
+        bool "Camera blink/eye closure only"
+    config FATIGUE_DETECTION_MODE_SILENCE
+        bool "Silence/no-speaking only"
+    config FATIGUE_DETECTION_MODE_BOTH
+        bool "Camera and silence"
+    config FATIGUE_DETECTION_MODE_OFF
+        bool "Off"
+endchoice
+
+config FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT
+    bool "Show camera debug preview by default"
+    depends on FATIGUE_DETECTION_MODE_CAMERA || FATIGUE_DETECTION_MODE_BOTH
+    default n
+    help
+        Show sampled camera frames while running local drowsiness detection.
+        This is useful for tuning, but it can cover the neutral standby image.
+
+endmenu
+
 choice
     depends on BOARD_TYPE_LILYGO_T_DISPLAY_P4
     prompt "Select the screen type"

main/application.cc

@@ -1,24 +1,88 @@
 #include "application.h"
+#include "assets.h"
+#include "assets/lang_config.h"
+#include "audio_codec.h"
 #include "board.h"
 #include "display.h"
-#include "system_info.h"
-#include "audio_codec.h"
-#include "mqtt_protocol.h"
-#include "websocket_protocol.h"
-#include "assets/lang_config.h"
 #include "mcp_server.h"
-#include "assets.h"
+#include "mqtt_protocol.h"
 #include "settings.h"
+#include "system_info.h"
+#include "websocket_protocol.h"
 
-#include <cstring>
-#include <esp_log.h>
-#include <cJSON.h>
 #include <driver/gpio.h>
+#include <esp_log.h>
+#include <sdkconfig.h>
 #include <arpa/inet.h>
+#include <cJSON.h>
 #include <font_awesome.h>
+#include <lwip/apps/sntp.h>
+#include <time.h>
+#include <cstdlib>
+#include <cstring>
 
 #define TAG "Application"
 
+namespace {
+constexpr const char* kDirectWebsocketUrl = "ws://172.19.0.240:8080";
+constexpr int kDirectWebsocketVersion = 3;
+constexpr bool kUseDirectWebsocketWithoutOta = true;
+constexpr int kDefaultFatigueListeningTimeoutSec = 12;
+constexpr int kDefaultFatigueIdleTimeoutSec = 12;
+constexpr int kDefaultFatigueCameraIntervalSec = 3;
+constexpr int kDefaultFatigueCameraClosedSamples = 2;
+constexpr int kDefaultFatigueCooldownSec = 60;
+constexpr const char* kDefaultFatigueEmotion = "wakeup";
+constexpr const char* kDefaultFatigueMessage = "你是不是又要睡着啦？快醒醒，我还要给你跳舞呢～";
+constexpr const char* kDefaultFatigueStatus = "打起精神";
+#if defined(CONFIG_FATIGUE_DETECTION_MODE_OFF)
+constexpr bool kDefaultFatigueEnabled = false;
+constexpr bool kDefaultFatigueCameraEnabled = false;
+constexpr bool kDefaultFatigueSilenceEnabled = false;
+#elif defined(CONFIG_FATIGUE_DETECTION_MODE_SILENCE)
+constexpr bool kDefaultFatigueEnabled = true;
+constexpr bool kDefaultFatigueCameraEnabled = false;
+constexpr bool kDefaultFatigueSilenceEnabled = true;
+#elif defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
+constexpr bool kDefaultFatigueEnabled = true;
+constexpr bool kDefaultFatigueCameraEnabled = true;
+constexpr bool kDefaultFatigueSilenceEnabled = true;
+#else
+constexpr bool kDefaultFatigueEnabled = true;
+constexpr bool kDefaultFatigueCameraEnabled = true;
+constexpr bool kDefaultFatigueSilenceEnabled = false;
+#endif
+#if defined(CONFIG_FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT)
+constexpr bool kDefaultFatigueCameraDebugPreview = true;
+#else
+constexpr bool kDefaultFatigueCameraDebugPreview = false;
+#endif
+
+void StartDirectTimeSync() {
+    setenv("TZ", "CST-8", 1);
+    tzset();
+
+    static bool sntp_started = false;
+    if (sntp_started) {
+        return;
+    }
+    sntp_started = true;
+
+    sntp_setoperatingmode(SNTP_OPMODE_POLL);
+    sntp_setservername(0, "ntp.aliyun.com");
+    sntp_init();
+}
+
+void ConfigureDirectWebsocket() {
+    Settings settings("websocket", true);
+    if (settings.GetString("url") != kDirectWebsocketUrl) {
+        settings.SetString("url", kDirectWebsocketUrl);
+    }
+    if (settings.GetInt("version") != kDirectWebsocketVersion) {
+        settings.SetInt("version", kDirectWebsocketVersion);
+    }
+}
+}  // namespace
 
 Application::Application() {
     event_group_ = xEventGroupCreate();
@@ -33,16 +97,16 @@ Application::Application() {
     aec_mode_ = kAecOff;
 #endif
 
-    esp_timer_create_args_t clock_timer_args = {
-        .callback = [](void* arg) {
-            Application* app = (Application*)arg;
-            xEventGroupSetBits(app->event_group_, MAIN_EVENT_CLOCK_TICK);
-        },
-        .arg = this,
-        .dispatch_method = ESP_TIMER_TASK,
-        .name = "clock_timer",
-        .skip_unhandled_events = true
-    };
+    esp_timer_create_args_t clock_timer_args = {.callback =
+                                                    [](void* arg) {
+                                                        Application* app = (Application*)arg;
+                                                        xEventGroupSetBits(app->event_group_,
+                                                                           MAIN_EVENT_CLOCK_TICK);
+                                                    },
+                                                .arg = this,
+                                                .dispatch_method = ESP_TIMER_TASK,
+                                                .name = "clock_timer",
+                                                .skip_unhandled_events = true};
     esp_timer_create(&clock_timer_args, &clock_timer_handle_);
 }
 
@@ -54,9 +118,7 @@ Application::~Application() {
     vEventGroupDelete(event_group_);
 }
 
-bool Application::SetDeviceState(DeviceState state) {
-    return state_machine_.TransitionTo(state);
-}
+bool Application::SetDeviceState(DeviceState state) { return state_machine_.TransitionTo(state); }
 
 void Application::Initialize() {
     auto& board = Board::GetInstance();
@@ -141,13 +203,16 @@ void Application::Initialize() {
                 display->SetStatus(Lang::Strings::DETECTING_MODULE);
                 break;
             case NetworkEvent::ModemErrorNoSim:
-                Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_PIN);
+                Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation",
+                      Lang::Sounds::OGG_ERR_PIN);
                 break;
             case NetworkEvent::ModemErrorRegDenied:
-                Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_REG);
+                Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation",
+                      Lang::Sounds::OGG_ERR_REG);
                 break;
             case NetworkEvent::ModemErrorInitFailed:
-                Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation", Lang::Sounds::OGG_EXCLAMATION);
+                Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation",
+                      Lang::Sounds::OGG_EXCLAMATION);
                 break;
             case NetworkEvent::ModemErrorTimeout:
                 display->SetStatus(Lang::Strings::REGISTERING_NETWORK);
@@ -167,18 +232,10 @@ void Application::Run() {
     vTaskPrioritySet(nullptr, 10);
 
     const EventBits_t ALL_EVENTS =
-        MAIN_EVENT_SCHEDULE |
-        MAIN_EVENT_SEND_AUDIO |
-        MAIN_EVENT_WAKE_WORD_DETECTED |
-        MAIN_EVENT_VAD_CHANGE |
-        MAIN_EVENT_CLOCK_TICK |
-        MAIN_EVENT_ERROR |
-        MAIN_EVENT_NETWORK_CONNECTED |
-        MAIN_EVENT_NETWORK_DISCONNECTED |
-        MAIN_EVENT_TOGGLE_CHAT |
-        MAIN_EVENT_START_LISTENING |
-        MAIN_EVENT_STOP_LISTENING |
-        MAIN_EVENT_ACTIVATION_DONE |
+        MAIN_EVENT_SCHEDULE | MAIN_EVENT_SEND_AUDIO | MAIN_EVENT_WAKE_WORD_DETECTED |
+        MAIN_EVENT_VAD_CHANGE | MAIN_EVENT_CLOCK_TICK | MAIN_EVENT_ERROR |
+        MAIN_EVENT_NETWORK_CONNECTED | MAIN_EVENT_NETWORK_DISCONNECTED | MAIN_EVENT_TOGGLE_CHAT |
+        MAIN_EVENT_START_LISTENING | MAIN_EVENT_STOP_LISTENING | MAIN_EVENT_ACTIVATION_DONE |
         MAIN_EVENT_STATE_CHANGED;
 
     while (true) {
@@ -186,7 +243,8 @@ void Application::Run() {
 
         if (bits & MAIN_EVENT_ERROR) {
             SetDeviceState(kDeviceStateIdle);
-            Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
+            Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark",
+                  Lang::Sounds::OGG_EXCLAMATION);
         }
 
         if (bits & MAIN_EVENT_NETWORK_CONNECTED) {
@@ -249,6 +307,7 @@ void Application::Run() {
             clock_ticks_++;
             auto display = Board::GetInstance().GetDisplay();
             display->UpdateStatusBar();
+            CheckFatigueReminder();
 
             // Print debug info every 10 seconds
             if (clock_ticks_ % 10 == 0) {
@@ -270,12 +329,14 @@ void Application::HandleNetworkConnectedEvent() {
             return;
         }
 
-        xTaskCreate([](void* arg) {
-            Application* app = static_cast<Application*>(arg);
-            app->ActivationTask();
-            app->activation_task_handle_ = nullptr;
-            vTaskDelete(NULL);
-        }, "activation", 4096 * 2, this, 2, &activation_task_handle_);
+        xTaskCreate(
+            [](void* arg) {
+                Application* app = static_cast<Application*>(arg);
+                app->ActivationTask();
+                app->activation_task_handle_ = nullptr;
+                vTaskDelete(NULL);
+            },
+            "activation", 4096 * 2, this, 2, &activation_task_handle_);
     }
 
     // Update the status bar immediately to show the network state
@@ -286,7 +347,8 @@ void Application::HandleNetworkConnectedEvent() {
 void Application::HandleNetworkDisconnectedEvent() {
     // Close current conversation when network disconnected
     auto state = GetDeviceState();
-    if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) {
+    if (state == kDeviceStateConnecting || state == kDeviceStateListening ||
+        state == kDeviceStateSpeaking) {
         ESP_LOGI(TAG, "Closing audio channel due to network disconnection");
         protocol_->CloseAudioChannel();
     }
@@ -324,6 +386,16 @@ void Application::ActivationTask() {
     // Create OTA object for activation process
     ota_ = std::make_unique<Ota>();
 
+    if (kUseDirectWebsocketWithoutOta) {
+        ConfigureDirectWebsocket();
+        StartDirectTimeSync();
+        ESP_LOGI(TAG, "Using direct websocket without OTA: %s", kDirectWebsocketUrl);
+        CheckAssetsVersion();
+        InitializeProtocol();
+        xEventGroupSetBits(event_group_, MAIN_EVENT_ACTIVATION_DONE);
+        return;
+    }
+
     // Check for new assets version
     CheckAssetsVersion();
 
@@ -362,7 +434,8 @@ void Application::CheckAssetsVersion() {
 
         char message[256];
         snprintf(message, sizeof(message), Lang::Strings::FOUND_NEW_ASSETS, download_url.c_str());
-        Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down", Lang::Sounds::OGG_UPGRADE);
+        Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down",
+              Lang::Sounds::OGG_UPGRADE);
 
         // Wait for the audio service to be idle for 3 seconds
         vTaskDelay(pdMS_TO_TICKS(3000));
@@ -370,19 +443,21 @@ void Application::CheckAssetsVersion() {
         board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
         display->SetChatMessage("system", Lang::Strings::PLEASE_WAIT);
 
-        bool success = assets.Download(download_url, [this, display](int progress, size_t speed) -> void {
-            char buffer[32];
-            snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024);
-            Schedule([display, message = std::string(buffer)]() {
-                display->SetChatMessage("system", message.c_str());
+        bool success =
+            assets.Download(download_url, [this, display](int progress, size_t speed) -> void {
+                char buffer[32];
+                snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024);
+                Schedule([display, message = std::string(buffer)]() {
+                    display->SetChatMessage("system", message.c_str());
+                });
             });
-        });
 
         board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
         vTaskDelay(pdMS_TO_TICKS(1000));
 
         if (!success) {
-            Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
+            Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark",
+                  Lang::Sounds::OGG_EXCLAMATION);
             vTaskDelay(pdMS_TO_TICKS(2000));
             SetDeviceState(kDeviceStateActivating);
             return;
@@ -398,7 +473,7 @@ void Application::CheckAssetsVersion() {
 void Application::CheckNewVersion() {
     const int MAX_RETRY = 10;
     int retry_count = 0;
-    int retry_delay = 10; // Initial retry delay in seconds
+    int retry_delay = 10;  // Initial retry delay in seconds
 
     auto& board = Board::GetInstance();
     while (true) {
@@ -414,27 +489,30 @@ void Application::CheckNewVersion() {
             }
 
             char error_message[128];
-            snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err, ota_->GetCheckVersionUrl().c_str());
+            snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err,
+                     ota_->GetCheckVersionUrl().c_str());
             char buffer[256];
-            snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay, error_message);
+            snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay,
+                     error_message);
             Alert(Lang::Strings::ERROR, buffer, "cloud_slash", Lang::Sounds::OGG_EXCLAMATION);
 
-            ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay, retry_count, MAX_RETRY);
+            ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay,
+                     retry_count, MAX_RETRY);
             for (int i = 0; i < retry_delay; i++) {
                 vTaskDelay(pdMS_TO_TICKS(1000));
                 if (GetDeviceState() == kDeviceStateIdle) {
                     break;
                 }
             }
-            retry_delay *= 2; // Double the retry delay
+            retry_delay *= 2;  // Double the retry delay
             continue;
         }
         retry_count = 0;
-        retry_delay = 10; // Reset retry delay
+        retry_delay = 10;  // Reset retry delay
 
         if (ota_->HasNewVersion()) {
             if (UpgradeFirmware(ota_->GetFirmwareUrl(), ota_->GetFirmwareVersion())) {
-                return; // This line will never be reached after reboot
+                return;  // This line will never be reached after reboot
             }
             // If upgrade failed, continue to normal operation
         }
@@ -477,18 +555,19 @@ void Application::InitializeProtocol() {
 
     display->SetStatus(Lang::Strings::LOADING_PROTOCOL);
 
+    Settings websocket_settings("websocket", false);
+    bool has_direct_websocket_config = !websocket_settings.GetString("url").empty();
+
     if (ota_->HasMqttConfig()) {
         protocol_ = std::make_unique<MqttProtocol>();
-    } else if (ota_->HasWebsocketConfig()) {
+    } else if (ota_->HasWebsocketConfig() || has_direct_websocket_config) {
         protocol_ = std::make_unique<WebsocketProtocol>();
     } else {
         ESP_LOGW(TAG, "No protocol specified in the OTA config, using MQTT");
         protocol_ = std::make_unique<MqttProtocol>();
     }
 
-    protocol_->OnConnected([this]() {
-        DismissAlert();
-    });
+    protocol_->OnConnected([this]() { DismissAlert(); });
 
     protocol_->OnNetworkError([this](const std::string& message) {
         last_error_message_ = message;
@@ -504,8 +583,10 @@ void Application::InitializeProtocol() {
     protocol_->OnAudioChannelOpened([this, codec, &board]() {
         board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
         if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
-            ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion",
-                protocol_->server_sample_rate(), codec->output_sample_rate());
+            ESP_LOGW(TAG,
+                     "Server sample rate %d does not match device output sample rate %d, "
+                     "resampling may cause distortion",
+                     protocol_->server_sample_rate(), codec->output_sample_rate());
         }
     });
 
@@ -573,9 +654,7 @@ void Application::InitializeProtocol() {
                 ESP_LOGI(TAG, "System command: %s", command->valuestring);
                 if (strcmp(command->valuestring, "reboot") == 0) {
                     // Do a reboot if user requests a OTA update
-                    Schedule([this]() {
-                        Reboot();
-                    });
+                    Schedule([this]() { Reboot(); });
                 } else {
                     ESP_LOGW(TAG, "Unknown system command: %s", command->valuestring);
                 }
@@ -585,7 +664,8 @@ void Application::InitializeProtocol() {
             auto message = cJSON_GetObjectItem(root, "message");
             auto emotion = cJSON_GetObjectItem(root, "emotion");
             if (cJSON_IsString(status) && cJSON_IsString(message) && cJSON_IsString(emotion)) {
-                Alert(status->valuestring, message->valuestring, emotion->valuestring, Lang::Sounds::OGG_VIBRATION);
+                Alert(status->valuestring, message->valuestring, emotion->valuestring,
+                      Lang::Sounds::OGG_VIBRATION);
             } else {
                 ESP_LOGW(TAG, "Alert command requires status, message and emotion");
             }
@@ -594,9 +674,10 @@ void Application::InitializeProtocol() {
             auto payload = cJSON_GetObjectItem(root, "payload");
             ESP_LOGI(TAG, "Received custom message: %s", cJSON_PrintUnformatted(root));
             if (cJSON_IsObject(payload)) {
-                Schedule([this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() {
-                    display->SetChatMessage("system", payload_str.c_str());
-                });
+                Schedule(
+                    [this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() {
+                        display->SetChatMessage("system", payload_str.c_str());
+                    });
             } else {
                 ESP_LOGW(TAG, "Invalid custom message format: missing payload");
             }
@@ -614,32 +695,27 @@ void Application::ShowActivationCode(const std::string& code, const std::string&
         char digit;
         const std::string_view& sound;
     };
-    static const std::array<digit_sound, 10> digit_sounds{{
-        digit_sound{'0', Lang::Sounds::OGG_0},
-        digit_sound{'1', Lang::Sounds::OGG_1}, 
-        digit_sound{'2', Lang::Sounds::OGG_2},
-        digit_sound{'3', Lang::Sounds::OGG_3},
-        digit_sound{'4', Lang::Sounds::OGG_4},
-        digit_sound{'5', Lang::Sounds::OGG_5},
-        digit_sound{'6', Lang::Sounds::OGG_6},
-        digit_sound{'7', Lang::Sounds::OGG_7},
-        digit_sound{'8', Lang::Sounds::OGG_8},
-        digit_sound{'9', Lang::Sounds::OGG_9}
-    }};
+    static const std::array<digit_sound, 10> digit_sounds{
+        {digit_sound{'0', Lang::Sounds::OGG_0}, digit_sound{'1', Lang::Sounds::OGG_1},
+         digit_sound{'2', Lang::Sounds::OGG_2}, digit_sound{'3', Lang::Sounds::OGG_3},
+         digit_sound{'4', Lang::Sounds::OGG_4}, digit_sound{'5', Lang::Sounds::OGG_5},
+         digit_sound{'6', Lang::Sounds::OGG_6}, digit_sound{'7', Lang::Sounds::OGG_7},
+         digit_sound{'8', Lang::Sounds::OGG_8}, digit_sound{'9', Lang::Sounds::OGG_9}}};
 
     // This sentence uses 9KB of SRAM, so we need to wait for it to finish
     Alert(Lang::Strings::ACTIVATION, message.c_str(), "link", Lang::Sounds::OGG_ACTIVATION);
 
     for (const auto& digit : code) {
         auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(),
-            [digit](const digit_sound& ds) { return ds.digit == digit; });
+                               [digit](const digit_sound& ds) { return ds.digit == digit; });
         if (it != digit_sounds.end()) {
             audio_service_.PlaySound(it->sound);
         }
     }
 }
 
-void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) {
+void Application::Alert(const char* status, const char* message, const char* emotion,
+                        const std::string_view& sound) {
     ESP_LOGW(TAG, "Alert [%s] %s: %s", emotion, status, message);
     auto display = Board::GetInstance().GetDisplay();
     display->SetStatus(status);
@@ -659,18 +735,192 @@ void Application::DismissAlert() {
     }
 }
 
-void Application::ToggleChatState() {
-    xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT);
+void Application::CheckFatigueReminder() {
+    auto state = GetDeviceState();
+    if (state != kDeviceStateListening) {
+        fatigue_silence_seconds_ = 0;
+        fatigue_reminder_triggered_in_listening_ = false;
+    }
+    if (state != kDeviceStateIdle) {
+        fatigue_idle_seconds_ = 0;
+    }
+
+    Settings settings("fatigue", false);
+    if (!settings.GetBool("enabled", kDefaultFatigueEnabled)) {
+        return;
+    }
+
+    int64_t now_us = esp_timer_get_time();
+    int cooldown_sec = settings.GetInt("cooldown_sec", kDefaultFatigueCooldownSec);
+    if (cooldown_sec < 10) {
+        cooldown_sec = 10;
+    } else if (cooldown_sec > 3600) {
+        cooldown_sec = 3600;
+    }
+    if (last_fatigue_reminder_time_us_ != 0 &&
+        now_us - last_fatigue_reminder_time_us_ < static_cast<int64_t>(cooldown_sec) * 1000000) {
+        return;
+    }
+
+    if (CheckCameraDrowsiness()) {
+        last_fatigue_reminder_time_us_ = esp_timer_get_time();
+        TriggerFatigueReminder();
+        return;
+    }
+
+    if (!settings.GetBool("silence_enabled", kDefaultFatigueSilenceEnabled)) {
+        fatigue_idle_seconds_ = 0;
+        fatigue_silence_seconds_ = 0;
+        fatigue_reminder_triggered_in_listening_ = false;
+        return;
+    }
+
+    if (state == kDeviceStateIdle) {
+        fatigue_idle_seconds_++;
+
+        int idle_timeout_sec = settings.GetInt("idle_timeout_sec", kDefaultFatigueIdleTimeoutSec);
+        if (idle_timeout_sec < 3) {
+            idle_timeout_sec = 3;
+        } else if (idle_timeout_sec > 3600) {
+            idle_timeout_sec = 3600;
+        }
+        if (fatigue_idle_seconds_ >= idle_timeout_sec) {
+            fatigue_idle_seconds_ = 0;
+            last_fatigue_reminder_time_us_ = now_us;
+            TriggerFatigueReminder();
+        }
+        return;
+    }
+
+    if (state != kDeviceStateListening) {
+        return;
+    }
+
+    if (audio_service_.IsVoiceDetected()) {
+        fatigue_silence_seconds_ = 0;
+        fatigue_reminder_triggered_in_listening_ = false;
+        return;
+    }
+
+    fatigue_silence_seconds_++;
+    if (fatigue_reminder_triggered_in_listening_) {
+        return;
+    }
+
+    int timeout_sec = settings.GetInt("listening_timeout_sec", kDefaultFatigueListeningTimeoutSec);
+    if (timeout_sec < 3) {
+        timeout_sec = 3;
+    } else if (timeout_sec > 300) {
+        timeout_sec = 300;
+    }
+    if (fatigue_silence_seconds_ < timeout_sec) {
+        return;
+    }
+
+    fatigue_reminder_triggered_in_listening_ = true;
+    last_fatigue_reminder_time_us_ = now_us;
+    TriggerFatigueReminder();
 }
 
-void Application::StartListening() {
-    xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING);
+bool Application::CheckCameraDrowsiness() {
+    auto state = GetDeviceState();
+    if (state != kDeviceStateIdle && state != kDeviceStateListening) {
+        fatigue_camera_closed_samples_ = 0;
+        return false;
+    }
+
+    Settings settings("fatigue", false);
+    if (!settings.GetBool("camera_enabled", kDefaultFatigueCameraEnabled)) {
+        fatigue_camera_closed_samples_ = 0;
+        return false;
+    }
+
+    int interval_sec = settings.GetInt("camera_interval_sec", kDefaultFatigueCameraIntervalSec);
+    if (interval_sec < 1) {
+        interval_sec = 1;
+    } else if (interval_sec > 60) {
+        interval_sec = 60;
+    }
+
+    int64_t now_us = esp_timer_get_time();
+    if (last_fatigue_camera_check_time_us_ != 0 &&
+        now_us - last_fatigue_camera_check_time_us_ <
+            static_cast<int64_t>(interval_sec) * 1000000) {
+        return false;
+    }
+    last_fatigue_camera_check_time_us_ = now_us;
+
+    auto camera = Board::GetInstance().GetCamera();
+    if (camera == nullptr) {
+        return false;
+    }
+
+    bool show_debug_preview =
+        settings.GetBool("camera_debug_preview", kDefaultFatigueCameraDebugPreview);
+    CameraDrowsinessResult result;
+    if (!camera->DetectDrowsiness(result, show_debug_preview)) {
+        fatigue_camera_closed_samples_ = 0;
+        return false;
+    }
+
+    ESP_LOGI(TAG, "Camera drowsiness: closed=%d score=%.2f baseline=%.2f",
+             result.eyes_closed ? 1 : 0, result.eye_openness_score, result.baseline_score);
+
+    if (result.eyes_closed) {
+        fatigue_camera_closed_samples_++;
+    } else {
+        fatigue_camera_closed_samples_ = 0;
+    }
+
+    int required_samples =
+        settings.GetInt("camera_closed_samples", kDefaultFatigueCameraClosedSamples);
+    if (required_samples < 1) {
+        required_samples = 1;
+    } else if (required_samples > 10) {
+        required_samples = 10;
+    }
+
+    if (fatigue_camera_closed_samples_ >= required_samples) {
+        fatigue_camera_closed_samples_ = 0;
+        return true;
+    }
+    return false;
 }
 
-void Application::StopListening() {
-    xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING);
+void Application::TriggerFatigueReminder() {
+    Settings settings("fatigue", false);
+    std::string emotion = settings.GetString("emotion", kDefaultFatigueEmotion);
+    std::string message = settings.GetString("message", kDefaultFatigueMessage);
+    std::string sound_asset = settings.GetString("sound_asset");
+
+    ESP_LOGW(TAG, "Fatigue reminder triggered: silence=%ds, emotion=%s", fatigue_silence_seconds_,
+             emotion.c_str());
+
+    auto display = Board::GetInstance().GetDisplay();
+    display->SetStatus(kDefaultFatigueStatus);
+    display->SetEmotion(emotion.c_str());
+    display->SetChatMessage("assistant", message.c_str());
+
+    if (!sound_asset.empty()) {
+        void* ptr = nullptr;
+        size_t size = 0;
+        auto& assets = Assets::GetInstance();
+        if (assets.partition_valid() && assets.GetAssetData(sound_asset, ptr, size)) {
+            audio_service_.PlaySound(std::string_view(static_cast<const char*>(ptr), size));
+            return;
+        }
+        ESP_LOGW(TAG, "Fatigue sound asset not found: %s", sound_asset.c_str());
+    }
+
+    audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
 }
 
+void Application::ToggleChatState() { xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT); }
+
+void Application::StartListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING); }
+
+void Application::StopListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING); }
+
 void Application::HandleToggleChatEvent() {
     auto state = GetDeviceState();
 
@@ -697,9 +947,7 @@ void Application::HandleToggleChatEvent() {
         if (!protocol_->IsAudioChannelOpened()) {
             SetDeviceState(kDeviceStateConnecting);
             // Schedule to let the state change be processed first (UI update)
-            Schedule([this, mode]() {
-                ContinueOpenAudioChannel(mode);
-            });
+            Schedule([this, mode]() { ContinueOpenAudioChannel(mode); });
             return;
         }
         SetListeningMode(mode);
@@ -746,9 +994,7 @@ void Application::HandleStartListeningEvent() {
         if (!protocol_->IsAudioChannelOpened()) {
             SetDeviceState(kDeviceStateConnecting);
             // Schedule to let the state change be processed first (UI update)
-            Schedule([this]() {
-                ContinueOpenAudioChannel(kListeningModeManualStop);
-            });
+            Schedule([this]() { ContinueOpenAudioChannel(kListeningModeManualStop); });
             return;
         }
         SetListeningMode(kListeningModeManualStop);
@@ -790,9 +1036,7 @@ void Application::HandleWakeWordDetectedEvent() {
             SetDeviceState(kDeviceStateConnecting);
             // Schedule to let the state change be processed first (UI update),
             // then continue with OpenAudioChannel which may block for ~1 second
-            Schedule([this, wake_word]() {
-                ContinueWakeWordInvoke(wake_word);
-            });
+            Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); });
             return;
         }
         // Channel already opened, continue directly
@@ -800,7 +1044,8 @@ void Application::HandleWakeWordDetectedEvent() {
     } else if (state == kDeviceStateSpeaking || state == kDeviceStateListening) {
         AbortSpeaking(kAbortReasonWakeWordDetected);
         // Clear send queue to avoid sending residues to server
-        while (audio_service_.PopPacketFromSendQueue());
+        while (audio_service_.PopPacketFromSendQueue())
+            ;
 
         if (state == kDeviceStateListening) {
             protocol_->SendStartListening(GetDefaultListeningMode());
@@ -865,24 +1110,26 @@ void Application::HandleStateChangedEvent() {
         case kDeviceStateUnknown:
         case kDeviceStateIdle:
             display->SetStatus(Lang::Strings::STANDBY);
-            display->ClearChatMessages();  // Clear messages first
-            display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count)
+            display->ClearChatMessages();    // Clear messages first
+            display->SetEmotion("neutral");  // Then set emotion (wechat mode checks child count)
             audio_service_.EnableVoiceProcessing(false);
             audio_service_.EnableWakeWordDetection(true);
             break;
         case kDeviceStateConnecting:
             display->SetStatus(Lang::Strings::CONNECTING);
-            display->SetEmotion("neutral");
             display->SetChatMessage("system", "");
+            display->SetEmotion("neutral");
             break;
         case kDeviceStateListening:
             display->SetStatus(Lang::Strings::LISTENING);
+            display->ClearChatMessages();
             display->SetEmotion("neutral");
 
             // Make sure the audio processor is running
             if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
-                // For auto mode, wait for playback queue to be empty before enabling voice processing
-                // This prevents audio truncation when STOP arrives late due to network jitter
+                // For auto mode, wait for playback queue to be empty before enabling voice
+                // processing This prevents audio truncation when STOP arrives late due to network
+                // jitter
                 if (listening_mode_ == kListeningModeAutoStop) {
                     audio_service_.WaitForPlaybackQueueEmpty();
                 }
@@ -978,7 +1225,8 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver
     }
     ESP_LOGI(TAG, "Starting firmware upgrade from URL: %s", upgrade_url.c_str());
 
-    Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download", Lang::Sounds::OGG_UPGRADE);
+    Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download",
+          Lang::Sounds::OGG_UPGRADE);
     vTaskDelay(pdMS_TO_TICKS(3000));
 
     SetDeviceState(kDeviceStateUpgrading);
@@ -1000,17 +1248,19 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver
 
     if (!upgrade_success) {
         // Upgrade failed, restart audio service and continue running
-        ESP_LOGE(TAG, "Firmware upgrade failed, restarting audio service and continuing operation...");
-        audio_service_.Start(); // Restart audio service
-        board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level
-        Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
+        ESP_LOGE(TAG,
+                 "Firmware upgrade failed, restarting audio service and continuing operation...");
+        audio_service_.Start();                              // Restart audio service
+        board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);  // Restore power save level
+        Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark",
+              Lang::Sounds::OGG_EXCLAMATION);
         vTaskDelay(pdMS_TO_TICKS(3000));
         return false;
     } else {
         // Upgrade success, reboot immediately
         ESP_LOGI(TAG, "Firmware upgrade successful, rebooting...");
         display->SetChatMessage("system", "Upgrade successful, rebooting...");
-        vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message
+        vTaskDelay(pdMS_TO_TICKS(1000));  // Brief pause to show message
         Reboot();
         return true;
     }
@@ -1029,17 +1279,13 @@ void Application::WakeWordInvoke(const std::string& wake_word) {
         if (!protocol_->IsAudioChannelOpened()) {
             SetDeviceState(kDeviceStateConnecting);
             // Schedule to let the state change be processed first (UI update)
-            Schedule([this, wake_word]() {
-                ContinueWakeWordInvoke(wake_word);
-            });
+            Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); });
             return;
         }
         // Channel already opened, continue directly
         ContinueWakeWordInvoke(wake_word);
     } else if (state == kDeviceStateSpeaking) {
-        Schedule([this]() {
-            AbortSpeaking(kAbortReasonNone);
-        });
+        Schedule([this]() { AbortSpeaking(kAbortReasonNone); });
     } else if (state == kDeviceStateListening) {
         Schedule([this]() {
             if (protocol_) {
@@ -1081,18 +1327,18 @@ void Application::SetAecMode(AecMode mode) {
         auto& board = Board::GetInstance();
         auto display = board.GetDisplay();
         switch (aec_mode_) {
-        case kAecOff:
-            audio_service_.EnableDeviceAec(false);
-            display->ShowNotification(Lang::Strings::RTC_MODE_OFF);
-            break;
-        case kAecOnServerSide:
-            audio_service_.EnableDeviceAec(false);
-            display->ShowNotification(Lang::Strings::RTC_MODE_ON);
-            break;
-        case kAecOnDeviceSide:
-            audio_service_.EnableDeviceAec(true);
-            display->ShowNotification(Lang::Strings::RTC_MODE_ON);
-            break;
+            case kAecOff:
+                audio_service_.EnableDeviceAec(false);
+                display->ShowNotification(Lang::Strings::RTC_MODE_OFF);
+                break;
+            case kAecOnServerSide:
+                audio_service_.EnableDeviceAec(false);
+                display->ShowNotification(Lang::Strings::RTC_MODE_ON);
+                break;
+            case kAecOnDeviceSide:
+                audio_service_.EnableDeviceAec(true);
+                display->ShowNotification(Lang::Strings::RTC_MODE_ON);
+                break;
         }
 
         // If the AEC mode is changed, close the audio channel
@@ -1102,9 +1348,7 @@ void Application::SetAecMode(AecMode mode) {
     });
 }
 
-void Application::PlaySound(const std::string_view& sound) {
-    audio_service_.PlaySound(sound);
-}
+void Application::PlaySound(const std::string_view& sound) { audio_service_.PlaySound(sound); }
 
 void Application::ResetProtocol() {
     Schedule([this]() {
@@ -1116,4 +1360,3 @@ void Application::ResetProtocol() {
         protocol_.reset();
     });
 }
-

main/application.h

@@ -140,6 +140,12 @@ private:
     bool aborted_ = false;
     bool assets_version_checked_ = false;
     bool play_popup_on_listening_ = false;  // Flag to play popup sound after state changes to listening
+    bool fatigue_reminder_triggered_in_listening_ = false;
+    int fatigue_silence_seconds_ = 0;
+    int fatigue_idle_seconds_ = 0;
+    int fatigue_camera_closed_samples_ = 0;
+    int64_t last_fatigue_reminder_time_us_ = 0;
+    int64_t last_fatigue_camera_check_time_us_ = 0;
     int clock_ticks_ = 0;
     TaskHandle_t activation_task_handle_ = nullptr;
 
@@ -155,6 +161,9 @@ private:
     void HandleWakeWordDetectedEvent();
     void ContinueOpenAudioChannel(ListeningMode mode);
     void ContinueWakeWordInvoke(const std::string& wake_word);
+    void CheckFatigueReminder();
+    bool CheckCameraDrowsiness();
+    void TriggerFatigueReminder();
 
     // Activation task (runs in background)
     void ActivationTask();

main/boards/common/camera.h

@@ -3,6 +3,13 @@
 
 #include <string>
 
+struct CameraDrowsinessResult {
+    bool valid = false;
+    bool eyes_closed = false;
+    float eye_openness_score = 0.0f;
+    float baseline_score = 0.0f;
+};
+
 class Camera {
 public:
     virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0;
@@ -10,6 +17,7 @@ public:
     virtual bool SetHMirror(bool enabled) = 0;
     virtual bool SetVFlip(bool enabled) = 0;
     virtual bool SetSwapBytes(bool enabled) { return false; }  // Optional, default no-op
+    virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) { return false; }
     virtual std::string Explain(const std::string& question) = 0;
 };
 

main/boards/common/esp32_camera.cc

@@ -5,6 +5,8 @@
 #include <cstring>
 #include <esp_log.h>
 #include <img_converters.h>
+#include <algorithm>
+#include <cmath>
 
 #include "esp32_camera.h"
 #include "board.h"
@@ -17,6 +19,38 @@
 
 #define TAG "Esp32Camera"
 
+namespace {
+constexpr float kClosedEyeScoreRatio = 0.58f;
+constexpr int kEyeSampleStep = 4;
+
+uint8_t Rgb565ToLuma(uint16_t pixel) {
+    uint8_t r = ((pixel >> 11) & 0x1f) << 3;
+    uint8_t g = ((pixel >> 5) & 0x3f) << 2;
+    uint8_t b = (pixel & 0x1f) << 3;
+    return static_cast<uint8_t>((static_cast<uint16_t>(r) * 30 +
+                                 static_cast<uint16_t>(g) * 59 +
+                                 static_cast<uint16_t>(b) * 11) / 100);
+}
+
+void DrawRectRgb565(uint16_t* pixels, int width, int height, int x0, int y0, int x1, int y1, uint16_t color) {
+    if (pixels == nullptr || width <= 0 || height <= 0) {
+        return;
+    }
+    x0 = std::clamp(x0, 0, width - 1);
+    x1 = std::clamp(x1, 0, width - 1);
+    y0 = std::clamp(y0, 0, height - 1);
+    y1 = std::clamp(y1, 0, height - 1);
+    for (int x = x0; x <= x1; x++) {
+        pixels[y0 * width + x] = color;
+        pixels[y1 * width + x] = color;
+    }
+    for (int y = y0; y <= y1; y++) {
+        pixels[y * width + x0] = color;
+        pixels[y * width + x1] = color;
+    }
+}
+} // namespace
+
 Esp32Camera::Esp32Camera(const camera_config_t &config) {
     esp_err_t err = esp_camera_init(&config);
     if (err != ESP_OK) {
@@ -152,6 +186,111 @@ bool Esp32Camera::SetSwapBytes(bool enabled) {
     return true;
 }
 
+bool Esp32Camera::DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview) {
+    result = {};
+
+    if (encoder_thread_.joinable()) {
+        encoder_thread_.join();
+    }
+    if (!streaming_on_) {
+        return false;
+    }
+
+    if (current_fb_) {
+        esp_camera_fb_return(current_fb_);
+        current_fb_ = nullptr;
+    }
+
+    camera_fb_t* fb = esp_camera_fb_get();
+    if (!fb) {
+        ESP_LOGW(TAG, "Camera drowsiness capture failed");
+        return false;
+    }
+
+    if (fb->format != PIXFORMAT_RGB565 || fb->width < 80 || fb->height < 80) {
+        ESP_LOGW(TAG, "Unsupported drowsiness frame: %dx%d format=%d",
+                 fb->width, fb->height, fb->format);
+        esp_camera_fb_return(fb);
+        return false;
+    }
+
+    const int width = fb->width;
+    const int height = fb->height;
+    const int x0 = width * 22 / 100;
+    const int x1 = width * 78 / 100;
+    const int y0 = height * 24 / 100;
+    const int y1 = height * 46 / 100;
+    const uint16_t* pixels = reinterpret_cast<const uint16_t*>(fb->buf);
+
+    if (show_debug_preview) {
+        size_t data_size = static_cast<size_t>(width) * static_cast<size_t>(height) * 2;
+        auto* preview_data = static_cast<uint8_t*>(heap_caps_malloc(data_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+        if (preview_data != nullptr) {
+            auto* dst = reinterpret_cast<uint16_t*>(preview_data);
+            const auto* src = reinterpret_cast<const uint16_t*>(fb->buf);
+            size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
+            for (size_t i = 0; i < pixel_count; i++) {
+                dst[i] = swap_bytes_enabled_ ? __builtin_bswap16(src[i]) : src[i];
+            }
+            DrawRectRgb565(dst, width, height, x0, y0, x1, y1, 0xF800);
+            auto display = dynamic_cast<LvglDisplay*>(Board::GetInstance().GetDisplay());
+            if (display != nullptr) {
+                display->SetPreviewImage(std::make_unique<LvglAllocatedImage>(
+                    preview_data, data_size, width, height, width * 2, LV_COLOR_FORMAT_RGB565));
+            } else {
+                heap_caps_free(preview_data);
+            }
+        }
+    }
+
+    float vertical_edge_sum = 0.0f;
+    float horizontal_edge_sum = 0.0f;
+    int samples = 0;
+    for (int y = y0; y + kEyeSampleStep < y1; y += kEyeSampleStep) {
+        for (int x = x0; x + kEyeSampleStep < x1; x += kEyeSampleStep) {
+            uint16_t p = pixels[y * width + x];
+            uint16_t px = pixels[y * width + x + kEyeSampleStep];
+            uint16_t py = pixels[(y + kEyeSampleStep) * width + x];
+            if (swap_bytes_enabled_) {
+                p = __builtin_bswap16(p);
+                px = __builtin_bswap16(px);
+                py = __builtin_bswap16(py);
+            }
+            uint8_t l = Rgb565ToLuma(p);
+            vertical_edge_sum += std::abs(static_cast<int>(l) - static_cast<int>(Rgb565ToLuma(py)));
+            horizontal_edge_sum += std::abs(static_cast<int>(l) - static_cast<int>(Rgb565ToLuma(px)));
+            samples++;
+        }
+    }
+
+    esp_camera_fb_return(fb);
+
+    if (samples == 0) {
+        return false;
+    }
+
+    // Open eyes usually keep more vertical texture in the fixed eye band.
+    // This is a lightweight central-face heuristic, not a landmark model.
+    float score = (vertical_edge_sum + horizontal_edge_sum * 0.35f) / samples;
+    if (eye_openness_baseline_ <= 0.0f) {
+        eye_openness_baseline_ = score;
+    } else if (score > eye_openness_baseline_ * 0.85f) {
+        eye_openness_baseline_ = eye_openness_baseline_ * 0.90f + score * 0.10f;
+    } else {
+        eye_openness_baseline_ = eye_openness_baseline_ * 0.995f + score * 0.005f;
+    }
+
+    result.valid = eye_openness_baseline_ > 1.0f;
+    result.eye_openness_score = score;
+    result.baseline_score = eye_openness_baseline_;
+    result.eyes_closed = result.valid && score < eye_openness_baseline_ * kClosedEyeScoreRatio;
+
+    ESP_LOGI(TAG, "Drowsiness frame=%dx%d eye_roi=(%d,%d)-(%d,%d) score=%.2f baseline=%.2f closed=%d",
+             width, height, x0, y0, x1, y1, result.eye_openness_score, result.baseline_score,
+             result.eyes_closed ? 1 : 0);
+    return result.valid;
+}
+
 std::string Esp32Camera::Explain(const std::string &question) {
     if (explain_url_.empty()) {
         throw std::runtime_error("Image explain URL or token is not set");

main/boards/common/esp32_camera.h

@@ -30,6 +30,7 @@ private:
     camera_fb_t *current_fb_ = nullptr;
     uint8_t *encode_buf_ = nullptr;  // Buffer for JPEG encoding (with optional byte swap)
     size_t encode_buf_size_ = 0;
+    float eye_openness_baseline_ = 0.0f;
 
 public:
     Esp32Camera(const camera_config_t &config);
@@ -40,5 +41,6 @@ public:
     virtual bool SetHMirror(bool enabled) override;
     virtual bool SetVFlip(bool enabled) override;
     virtual bool SetSwapBytes(bool enabled) override;
+    virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) override;
     virtual std::string Explain(const std::string &question) override;
 };

main/boards/zhengchen-cam/custom-emojis/README.md

@@ -0,0 +1,62 @@
+# Custom Emojis
+
+Put your custom PNG files in `png/` and custom GIF files in `gif/`.
+
+The filename without extension is used as the emotion name. Directory names are
+not part of the emotion name, so `png/neutral.png` is loaded as `neutral`.
+
+The display code looks up images by names such as:
+
+- `neutral.png` or `neutral.gif`
+- `happy.png` or `happy.gif`
+- `sad.png` or `sad.gif`
+- `angry.png` or `angry.gif`
+- `thinking.png` or `thinking.gif`
+- `confused.png` or `confused.gif`
+- `surprised.png` or `surprised.gif`
+- `shocked.png` or `shocked.gif`
+- `sleepy.png` or `sleepy.gif`
+- `relaxed.png` or `relaxed.gif`
+
+Recommended minimum set:
+
+- `neutral`
+- `happy`
+- `thinking`
+- `sad`
+- `angry`
+
+Fatigue reminder:
+
+- Add `wakeup.gif` or `wakeup.png` to make the idle-fatigue reminder show a custom idol animation.
+- The reminder defaults to `wakeup` when camera drowsiness is detected, then waits 60 seconds before it can trigger again.
+- Optional NVS settings in namespace `fatigue`:
+  - `enabled` (`bool`, default `true`)
+  - `camera_enabled` (`bool`, default `true`)
+  - `camera_debug_preview` (`bool`, default `true`; shows the sampled frame with the eye ROI box)
+  - `camera_interval_sec` (`int`, default `3`)
+  - `camera_closed_samples` (`int`, default `2`)
+  - `silence_enabled` (`bool`, default `false`)
+  - `idle_timeout_sec` (`int`, default `12`)
+  - `listening_timeout_sec` (`int`, default `12`)
+  - `cooldown_sec` (`int`, default `60`)
+  - `emotion` (`string`, default `wakeup`)
+  - `message` (`string`, default Chinese wake-up line)
+  - `sound_asset` (`string`, optional OGG filename in the assets partition)
+
+If an emotion-specific image is missing, the firmware falls back to `neutral`
+before using the built-in icon.
+
+For Zhengchen CAM boards, the build packages both subdirectories:
+
+- Put static/default faces in `png/`, such as `png/neutral.png`.
+- Put animated/special actions in `gif/`, such as `gif/wakeup.gif`.
+- If both folders contain the same emotion name, PNG wins. For example,
+  `png/neutral.png` is used before `gif/neutral.gif`.
+
+After adding or replacing files, run a full flash so the assets partition is
+updated:
+
+```bash
+idf.py flash
+```

main/boards/zhengchen-cam/mcp_controller.cc

@@ -1,5 +1,6 @@
 #include <cJSON.h>
 #include <esp_log.h>
+#include <sdkconfig.h>
 
 #include <cstring>
 
@@ -13,6 +14,52 @@
 
 #define TAG "MCPController"
 
+namespace {
+constexpr bool kDefaultFatigueEnabled =
+#if defined(CONFIG_FATIGUE_DETECTION_MODE_OFF)
+    false;
+#else
+    true;
+#endif
+
+constexpr bool kDefaultFatigueCameraEnabled =
+#if defined(CONFIG_FATIGUE_DETECTION_MODE_CAMERA) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
+    true;
+#else
+    false;
+#endif
+
+constexpr bool kDefaultFatigueSilenceEnabled =
+#if defined(CONFIG_FATIGUE_DETECTION_MODE_SILENCE) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
+    true;
+#else
+    false;
+#endif
+
+constexpr bool kDefaultFatigueCameraDebugPreview =
+#if defined(CONFIG_FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT)
+    true;
+#else
+    false;
+#endif
+
+std::string GetFatigueDetectionMode(bool enabled, bool camera_enabled, bool silence_enabled) {
+    if (!enabled) {
+        return "off";
+    }
+    if (camera_enabled && silence_enabled) {
+        return "both";
+    }
+    if (camera_enabled) {
+        return "camera";
+    }
+    if (silence_enabled) {
+        return "silence";
+    }
+    return "off";
+}
+}  // namespace
+
 class MCPController {
 public:
     MCPController() {
@@ -42,7 +89,6 @@ public:
                 app.SetAecMode(kAecOff);
                 return "{\"success\": true, \"message\": \"AEC对话打断模式已关闭\"}";
             }else {
-                auto& board = Board::GetInstance();
                 app.SetAecMode(kAecOnDeviceSide);
                 
                 return "{\"success\": true, \"message\": \"AEC对话打断模式已开启\"}";
@@ -79,6 +125,79 @@ public:
         }
     );
 
+    mcp_server.AddTool(
+        "self.fatigue.set_detection_mode",
+        "切换疲劳提醒检测模式。当用户想切换打瞌睡检测、眨眼检测、不讲话检测、关闭疲劳提醒或查看摄像头取样画面时使用此工具。\n"
+        "参数：\n"
+        "   `mode`: 检测模式，可选值只有 `camera`(只用摄像头眨眼/闭眼检测)、`silence`(只用不讲话/静默检测)、`both`(两个都开)、`off`(关闭疲劳提醒)\n"
+        "   `debug_preview`: 是否显示摄像头调试预览。默认 false；只有用户明确想看摄像头拍到什么时才设为 true\n"
+        "返回值：\n"
+        "   反馈当前疲劳检测模式，不需要确认，立即播报相关数据\n",
+        PropertyList({
+            Property("mode", kPropertyTypeString),
+            Property("debug_preview", kPropertyTypeBoolean, false),
+        }),
+        [](const PropertyList& properties) -> ReturnValue {
+            auto mode = properties["mode"].value<std::string>();
+            bool debug_preview = properties["debug_preview"].value<bool>();
+
+            bool enabled = true;
+            bool camera_enabled = false;
+            bool silence_enabled = false;
+            std::string message;
+
+            if (mode == "camera") {
+                camera_enabled = true;
+                message = "已切换为摄像头眨眼检测模式";
+            } else if (mode == "silence") {
+                silence_enabled = true;
+                message = "已切换为不讲话静默检测模式";
+            } else if (mode == "both") {
+                camera_enabled = true;
+                silence_enabled = true;
+                message = "已切换为摄像头加静默双检测模式";
+            } else if (mode == "off") {
+                enabled = false;
+                message = "疲劳提醒检测已关闭";
+            } else {
+                return "{\"success\": false, \"message\": \"检测模式无效，只能使用 camera、silence、both 或 off\"}";
+            }
+
+            Settings settings("fatigue", true);
+            settings.SetBool("enabled", enabled);
+            settings.SetBool("camera_enabled", camera_enabled);
+            settings.SetBool("silence_enabled", silence_enabled);
+            settings.SetBool("camera_debug_preview", debug_preview);
+
+            return std::string("{\"success\": true, \"mode\": \"") + mode +
+                   "\", \"debug_preview\": " + (debug_preview ? "true" : "false") +
+                   ", \"message\": \"" + message + "\"}";
+        }
+    );
+
+    mcp_server.AddTool(
+        "self.fatigue.get_detection_mode",
+        "获取当前疲劳提醒检测模式。当用户询问现在使用哪种打瞌睡检测模式时使用此工具。\n"
+        "返回值：\n"
+        "   当前检测模式和摄像头预览状态\n",
+        PropertyList(),
+        [](const PropertyList&) -> ReturnValue {
+            Settings settings("fatigue", false);
+            bool enabled = settings.GetBool("enabled", kDefaultFatigueEnabled);
+            bool camera_enabled = settings.GetBool("camera_enabled", kDefaultFatigueCameraEnabled);
+            bool silence_enabled =
+                settings.GetBool("silence_enabled", kDefaultFatigueSilenceEnabled);
+            bool debug_preview =
+                settings.GetBool("camera_debug_preview", kDefaultFatigueCameraDebugPreview);
+            auto mode = GetFatigueDetectionMode(enabled, camera_enabled, silence_enabled);
+
+            return std::string("{\"success\": true, \"mode\": \"") + mode +
+                   "\", \"camera_enabled\": " + (camera_enabled ? "true" : "false") +
+                   ", \"silence_enabled\": " + (silence_enabled ? "true" : "false") +
+                   ", \"debug_preview\": " + (debug_preview ? "true" : "false") + "}";
+        }
+    );
+
         ESP_LOGI(TAG, "MCP工具注册完成");
     }
 

main/display/lcd_display.cc

@@ -22,6 +22,60 @@ LV_FONT_DECLARE(BUILTIN_TEXT_FONT);
 LV_FONT_DECLARE(BUILTIN_ICON_FONT);
 LV_FONT_DECLARE(font_awesome_30_4);
 
+namespace {
+constexpr int kEmojiMaxScale = 1024;
+
+lv_coord_t ObjectHeight(lv_obj_t* obj) {
+    if (obj == nullptr) {
+        return 0;
+    }
+    if (lv_obj_has_flag(obj, LV_OBJ_FLAG_HIDDEN)) {
+        return 0;
+    }
+    lv_obj_update_layout(obj);
+    return lv_obj_get_height(obj);
+}
+
+void ApplyEmojiImageScale(lv_obj_t* image_obj, lv_obj_t* image_box, const lv_image_dsc_t* image_dsc,
+                          lv_coord_t top_reserved, lv_coord_t bottom_reserved) {
+    if (image_obj == nullptr || image_dsc == nullptr) {
+        return;
+    }
+
+    lv_coord_t image_width = image_dsc->header.w;
+    lv_coord_t image_height = image_dsc->header.h;
+    if (image_width <= 0 || image_height <= 0) {
+        lv_image_set_scale(image_obj, 256);
+        return;
+    }
+
+    lv_coord_t max_width = LV_HOR_RES;
+    lv_coord_t max_height = LV_VER_RES - top_reserved - bottom_reserved;
+    max_height = std::max<lv_coord_t>(max_height, 1);
+    lv_coord_t scale_w = max_width * 256 / image_width;
+    lv_coord_t scale_h = max_height * 256 / image_height;
+    lv_coord_t scale = std::min(scale_w, scale_h);
+    scale = std::min<lv_coord_t>(scale, kEmojiMaxScale);
+    scale = std::max<lv_coord_t>(scale, 1);
+    lv_coord_t scaled_width = image_width * scale / 256;
+    lv_coord_t scaled_height = image_height * scale / 256;
+
+    lv_image_set_scale(image_obj, scale);
+    lv_obj_set_size(image_obj, scaled_width, scaled_height);
+    lv_obj_t* align_obj = image_box != nullptr ? image_box : image_obj;
+    lv_obj_set_size(align_obj, scaled_width, scaled_height);
+    lv_obj_align(align_obj, LV_ALIGN_TOP_MID, 0, top_reserved + (max_height - scaled_height) / 2);
+    if (image_box != nullptr) {
+        lv_obj_center(image_obj);
+    }
+    ESP_LOGD(TAG, "Emoji image scale=%ld reserved=%ld/%ld size=%ldx%ld -> %ldx%ld",
+             static_cast<long>(scale), static_cast<long>(top_reserved),
+             static_cast<long>(bottom_reserved), static_cast<long>(image_width),
+             static_cast<long>(image_height),
+             static_cast<long>(scaled_width), static_cast<long>(scaled_height));
+}
+}  // namespace
+
 void LcdDisplay::InitializeLcdThemes() {
     auto text_font = std::make_shared<LvglBuiltInFont>(&BUILTIN_TEXT_FONT);
     auto icon_font = std::make_shared<LvglBuiltInFont>(&BUILTIN_ICON_FONT);
@@ -1136,6 +1190,9 @@ void LcdDisplay::SetEmotion(const char* emotion) {
 
     auto emoji_collection = static_cast<LvglTheme*>(current_theme_)->emoji_collection();
     auto image = emoji_collection != nullptr ? emoji_collection->GetEmojiImage(emotion) : nullptr;
+    if (image == nullptr && emoji_collection != nullptr && strcmp(emotion, "neutral") != 0) {
+        image = emoji_collection->GetEmojiImage("neutral");
+    }
     if (image == nullptr) {
         const char* utf8 = font_awesome_get_utf8(emotion);
         if (utf8 != nullptr && emoji_label_ != nullptr) {
@@ -1148,17 +1205,37 @@ void LcdDisplay::SetEmotion(const char* emotion) {
     }
 
     DisplayLockGuard lock(this);
+    bool use_full_screen_center = strcmp(emotion, "neutral") == 0;
+    if (use_full_screen_center && preview_image_ != nullptr) {
+        esp_timer_stop(preview_timer_);
+        if (emoji_box_ != nullptr) {
+            lv_obj_remove_flag(emoji_box_, LV_OBJ_FLAG_HIDDEN);
+        }
+        lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
+        preview_image_cached_.reset();
+    }
+    lv_coord_t top_reserved = use_full_screen_center ? 0 : ObjectHeight(top_bar_);
+    lv_coord_t bottom_reserved = use_full_screen_center ? 0 : ObjectHeight(status_bar_) + ObjectHeight(bottom_bar_);
+    auto apply_emoji_layout = [this, top_reserved, bottom_reserved](const lv_image_dsc_t* image_dsc) {
+        ApplyEmojiImageScale(emoji_image_, emoji_box_, image_dsc, top_reserved, bottom_reserved);
+    };
+
     if (image->IsGif()) {
         // Create new GIF controller
         gif_controller_ = std::make_unique<LvglGif>(image->image_dsc());
 
         if (gif_controller_->IsLoaded()) {
             // Set up frame update callback
-            gif_controller_->SetFrameCallback(
-                [this]() { lv_image_set_src(emoji_image_, gif_controller_->image_dsc()); });
+            gif_controller_->SetFrameCallback([this, apply_emoji_layout]() {
+                auto frame = gif_controller_->image_dsc();
+                lv_image_set_src(emoji_image_, frame);
+                apply_emoji_layout(frame);
+            });
 
             // Set initial frame and start animation
-            lv_image_set_src(emoji_image_, gif_controller_->image_dsc());
+            auto frame = gif_controller_->image_dsc();
+            lv_image_set_src(emoji_image_, frame);
+            apply_emoji_layout(frame);
             gif_controller_->Start();
 
             // Show GIF, hide others
@@ -1169,7 +1246,9 @@ void LcdDisplay::SetEmotion(const char* emotion) {
             gif_controller_.reset();
         }
     } else {
-        lv_image_set_src(emoji_image_, image->image_dsc());
+        auto image_dsc = image->image_dsc();
+        lv_image_set_src(emoji_image_, image_dsc);
+        apply_emoji_layout(image_dsc);
         lv_obj_add_flag(emoji_label_, LV_OBJ_FLAG_HIDDEN);
         lv_obj_remove_flag(emoji_image_, LV_OBJ_FLAG_HIDDEN);
     }

main/display/lvgl_display/lvgl_display.cc

@@ -3,6 +3,7 @@
 #include <string>
 #include <cstdlib>
 #include <cstring>
+#include <ctime>
 #include <font_awesome.h>
 
 #include "lvgl_display.h"
@@ -22,6 +23,21 @@ LvglDisplay::LvglDisplay() {
             LvglDisplay *display = static_cast<LvglDisplay*>(arg);
             DisplayLockGuard lock(display);
             lv_obj_add_flag(display->notification_label_, LV_OBJ_FLAG_HIDDEN);
+            if (Application::GetInstance().GetDeviceState() == kDeviceStateIdle && display->time_label_ != nullptr) {
+                time_t now = time(NULL);
+                struct tm* tm = localtime(&now);
+                if (tm->tm_year >= 2025 - 1900) {
+                    char time_str[16];
+                    strftime(time_str, sizeof(time_str), "%H:%M", tm);
+                    lv_label_set_text(display->time_label_, time_str);
+                    lv_obj_remove_flag(display->time_label_, LV_OBJ_FLAG_HIDDEN);
+                    lv_obj_add_flag(display->status_label_, LV_OBJ_FLAG_HIDDEN);
+                    return;
+                }
+            }
+            if (display->time_label_ != nullptr) {
+                lv_obj_add_flag(display->time_label_, LV_OBJ_FLAG_HIDDEN);
+            }
             lv_obj_remove_flag(display->status_label_, LV_OBJ_FLAG_HIDDEN);
         },
         .arg = this,

scripts/build_default_assets.py

@@ -235,8 +235,14 @@ def process_emoji_collection(emoji_collection_dir, assets_dir):
         "buxue": ["thinking", "confused", "embarrassed"]
     }
     
-    # Copy each image from input directory to build/assets directory
+    seen_emoji_names = set()
+
+    # Copy each image from input directory to build/assets directory. Prefer PNG
+    # over GIF for duplicate emotion names so static defaults can coexist with
+    # animated special actions in sibling directories.
     for root, dirs, files in os.walk(emoji_collection_dir):
+        dirs.sort(key=lambda d: (0 if d == "png" else 1 if d == "gif" else 2, d))
+        files.sort(key=lambda f: (0 if f.lower().endswith(".png") else 1, f))
         for file in files:
             if file.lower().endswith(('.png', '.gif')):
                 # Copy file
@@ -245,6 +251,9 @@ def process_emoji_collection(emoji_collection_dir, assets_dir):
                 if copy_file(src_file, dst_file):
                     # Get filename without extension
                     filename_without_ext = os.path.splitext(file)[0]
+                    if filename_without_ext in seen_emoji_names:
+                        continue
+                    seen_emoji_names.add(filename_without_ext)
                     
                     # Add main emoji entry
                     emoji_list.append({
@@ -715,10 +724,21 @@ def get_emoji_collection_path(default_emoji_collection, xiaozhi_fonts_path, proj
     - PNG emoji collections from xiaozhi-fonts (e.g., emojis_32, twemoji_64)
     - GIF emoji collections from xiaozhi-fonts (e.g., noto-emoji_128, noto-emoji_64)
     - Otto GIF emoji collection (otto-gif)
+    - Custom project-relative or absolute directory paths
     """
     if not default_emoji_collection:
         return None
 
+    candidate_paths = []
+    if os.path.isabs(default_emoji_collection):
+        candidate_paths.append(default_emoji_collection)
+    elif project_root:
+        candidate_paths.append(os.path.join(project_root, default_emoji_collection))
+
+    for candidate_path in candidate_paths:
+        if os.path.isdir(candidate_path):
+            return candidate_path
+    
     # Special handling for otto-gif collection
     if default_emoji_collection == 'otto-gif':
         if project_root: