verachen/kedia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: cam and slience detect sleep

Browse Source
This commit is contained in:
0Xiao0
2026-06-17 15:02:16 +08:00
parent 154ce461d7
commit a8c6c62c92
9 changed files with 564 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
main/Kconfig.projbuild
View File

@ -141,6 +141,36 @@ config ZHENGCHEN_CAM_USE_GIF_EMOJI
When enabled, default assets use custom-emojis/gif. When enabled, default assets use custom-emojis/gif.
When disabled, default assets use custom-emojis/png. When disabled, default assets use custom-emojis/png.
menu "Fatigue Reminder"
depends on BOARD_TYPE_ZHENGCHEN_CAM || BOARD_TYPE_ZHENGCHEN_CAM_ML307
choice FATIGUE_DETECTION_DEFAULT_MODE
prompt "Default fatigue detection mode"
default FATIGUE_DETECTION_MODE_CAMERA
help
Select the default fatigue reminder detection mode used when no runtime NVS
override exists.
config FATIGUE_DETECTION_MODE_CAMERA
bool "Camera blink/eye closure only"
config FATIGUE_DETECTION_MODE_SILENCE
bool "Silence/no-speaking only"
config FATIGUE_DETECTION_MODE_BOTH
bool "Camera and silence"
config FATIGUE_DETECTION_MODE_OFF
bool "Off"
endchoice
config FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT
bool "Show camera debug preview by default"
depends on FATIGUE_DETECTION_MODE_CAMERA || FATIGUE_DETECTION_MODE_BOTH
default n
help
Show sampled camera frames while running local drowsiness detection.
This is useful for tuning, but it can cover the neutral standby image.
endmenu
choice choice
depends on BOARD_TYPE_LILYGO_T_DISPLAY_P4 depends on BOARD_TYPE_LILYGO_T_DISPLAY_P4
prompt "Select the screen type" prompt "Select the screen type"

407
main/application.cc
View File

@ -1,24 +1,25 @@
#include "application.h" #include "application.h"
#include "assets.h"
#include "assets/lang_config.h"
#include "audio_codec.h"
#include "board.h" #include "board.h"
#include "display.h" #include "display.h"
#include "system_info.h"
#include "audio_codec.h"
#include "mqtt_protocol.h"
#include "websocket_protocol.h"
#include "assets/lang_config.h"
#include "mcp_server.h" #include "mcp_server.h"
#include "assets.h" #include "mqtt_protocol.h"
#include "settings.h" #include "settings.h"
#include "system_info.h"
#include "websocket_protocol.h"
#include <cstring>
#include <esp_log.h>
#include <cJSON.h>
#include <driver/gpio.h> #include <driver/gpio.h>
#include <esp_log.h>
#include <sdkconfig.h>
#include <arpa/inet.h> #include <arpa/inet.h>
#include <cJSON.h>
#include <font_awesome.h> #include <font_awesome.h>
#include <lwip/apps/sntp.h> #include <lwip/apps/sntp.h>
#include <time.h> #include <time.h>
#include <cstdlib> #include <cstdlib>
#include <cstring>
#define TAG "Application" #define TAG "Application"
@ -28,10 +29,34 @@ constexpr int kDirectWebsocketVersion = 3;
constexpr bool kUseDirectWebsocketWithoutOta = true; constexpr bool kUseDirectWebsocketWithoutOta = true;
constexpr int kDefaultFatigueListeningTimeoutSec = 12; constexpr int kDefaultFatigueListeningTimeoutSec = 12;
constexpr int kDefaultFatigueIdleTimeoutSec = 12; constexpr int kDefaultFatigueIdleTimeoutSec = 12;
constexpr int kDefaultFatigueCameraIntervalSec = 3;
constexpr int kDefaultFatigueCameraClosedSamples = 2;
constexpr int kDefaultFatigueCooldownSec = 60; constexpr int kDefaultFatigueCooldownSec = 60;
constexpr const char* kDefaultFatigueEmotion = "wakeup"; constexpr const char* kDefaultFatigueEmotion = "wakeup";
constexpr const char* kDefaultFatigueMessage = "你是不是又要睡着啦？快醒醒，我还要给你跳舞呢～"; constexpr const char* kDefaultFatigueMessage = "你是不是又要睡着啦？快醒醒，我还要给你跳舞呢～";
constexpr const char* kDefaultFatigueStatus = "打起精神"; constexpr const char* kDefaultFatigueStatus = "打起精神";
#if defined(CONFIG_FATIGUE_DETECTION_MODE_OFF)
constexpr bool kDefaultFatigueEnabled = false;
constexpr bool kDefaultFatigueCameraEnabled = false;
constexpr bool kDefaultFatigueSilenceEnabled = false;
#elif defined(CONFIG_FATIGUE_DETECTION_MODE_SILENCE)
constexpr bool kDefaultFatigueEnabled = true;
constexpr bool kDefaultFatigueCameraEnabled = false;
constexpr bool kDefaultFatigueSilenceEnabled = true;
#elif defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
constexpr bool kDefaultFatigueEnabled = true;
constexpr bool kDefaultFatigueCameraEnabled = true;
constexpr bool kDefaultFatigueSilenceEnabled = true;
#else
constexpr bool kDefaultFatigueEnabled = true;
constexpr bool kDefaultFatigueCameraEnabled = true;
constexpr bool kDefaultFatigueSilenceEnabled = false;
#endif
#if defined(CONFIG_FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT)
constexpr bool kDefaultFatigueCameraDebugPreview = true;
#else
constexpr bool kDefaultFatigueCameraDebugPreview = false;
#endif
void StartDirectTimeSync() { void StartDirectTimeSync() {
setenv("TZ", "CST-8", 1); setenv("TZ", "CST-8", 1);
@ -57,8 +82,7 @@ void ConfigureDirectWebsocket() {
settings.SetInt("version", kDirectWebsocketVersion); settings.SetInt("version", kDirectWebsocketVersion);
} }
} }
} // namespace } // namespace
Application::Application() { Application::Application() {
event_group_ = xEventGroupCreate(); event_group_ = xEventGroupCreate();
@ -73,16 +97,16 @@ Application::Application() {
aec_mode_ = kAecOff; aec_mode_ = kAecOff;
#endif #endif
esp_timer_create_args_t clock_timer_args = { esp_timer_create_args_t clock_timer_args = {.callback =
.callback = [](void* arg) { [](void* arg) {
Application* app = (Application*)arg; Application* app = (Application*)arg;
xEventGroupSetBits(app->event_group_, MAIN_EVENT_CLOCK_TICK); xEventGroupSetBits(app->event_group_,
}, MAIN_EVENT_CLOCK_TICK);
.arg = this, },
.dispatch_method = ESP_TIMER_TASK, .arg = this,
.name = "clock_timer", .dispatch_method = ESP_TIMER_TASK,
.skip_unhandled_events = true .name = "clock_timer",
}; .skip_unhandled_events = true};
esp_timer_create(&clock_timer_args, &clock_timer_handle_); esp_timer_create(&clock_timer_args, &clock_timer_handle_);
} }
@ -94,9 +118,7 @@ Application::~Application() {
vEventGroupDelete(event_group_); vEventGroupDelete(event_group_);
} }
bool Application::SetDeviceState(DeviceState state) { bool Application::SetDeviceState(DeviceState state) { return state_machine_.TransitionTo(state); }
return state_machine_.TransitionTo(state);
}
void Application::Initialize() { void Application::Initialize() {
auto& board = Board::GetInstance(); auto& board = Board::GetInstance();
@ -141,7 +163,7 @@ void Application::Initialize() {
// Set network event callback for UI updates and network state handling // Set network event callback for UI updates and network state handling
board.SetNetworkEventCallback([this](NetworkEvent event, const std::string& data) { board.SetNetworkEventCallback([this](NetworkEvent event, const std::string& data) {
auto display = Board::GetInstance().GetDisplay(); auto display = Board::GetInstance().GetDisplay();
switch (event) { switch (event) {
case NetworkEvent::Scanning: case NetworkEvent::Scanning:
display->ShowNotification(Lang::Strings::SCANNING_WIFI, 30000); display->ShowNotification(Lang::Strings::SCANNING_WIFI, 30000);
@ -181,13 +203,16 @@ void Application::Initialize() {
display->SetStatus(Lang::Strings::DETECTING_MODULE); display->SetStatus(Lang::Strings::DETECTING_MODULE);
break; break;
case NetworkEvent::ModemErrorNoSim: case NetworkEvent::ModemErrorNoSim:
Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_PIN); Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation",
Lang::Sounds::OGG_ERR_PIN);
break; break;
case NetworkEvent::ModemErrorRegDenied: case NetworkEvent::ModemErrorRegDenied:
Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_REG); Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation",
Lang::Sounds::OGG_ERR_REG);
break; break;
case NetworkEvent::ModemErrorInitFailed: case NetworkEvent::ModemErrorInitFailed:
Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation", Lang::Sounds::OGG_EXCLAMATION); Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation",
Lang::Sounds::OGG_EXCLAMATION);
break; break;
case NetworkEvent::ModemErrorTimeout: case NetworkEvent::ModemErrorTimeout:
display->SetStatus(Lang::Strings::REGISTERING_NETWORK); display->SetStatus(Lang::Strings::REGISTERING_NETWORK);
@ -206,19 +231,11 @@ void Application::Run() {
// Set the priority of the main task to 10 // Set the priority of the main task to 10
vTaskPrioritySet(nullptr, 10); vTaskPrioritySet(nullptr, 10);
const EventBits_t ALL_EVENTS = const EventBits_t ALL_EVENTS =
MAIN_EVENT_SCHEDULE | MAIN_EVENT_SCHEDULE | MAIN_EVENT_SEND_AUDIO | MAIN_EVENT_WAKE_WORD_DETECTED |
MAIN_EVENT_SEND_AUDIO | MAIN_EVENT_VAD_CHANGE | MAIN_EVENT_CLOCK_TICK | MAIN_EVENT_ERROR |
MAIN_EVENT_WAKE_WORD_DETECTED | MAIN_EVENT_NETWORK_CONNECTED | MAIN_EVENT_NETWORK_DISCONNECTED | MAIN_EVENT_TOGGLE_CHAT |
MAIN_EVENT_VAD_CHANGE | MAIN_EVENT_START_LISTENING | MAIN_EVENT_STOP_LISTENING | MAIN_EVENT_ACTIVATION_DONE |
MAIN_EVENT_CLOCK_TICK |
MAIN_EVENT_ERROR |
MAIN_EVENT_NETWORK_CONNECTED |
MAIN_EVENT_NETWORK_DISCONNECTED |
MAIN_EVENT_TOGGLE_CHAT |
MAIN_EVENT_START_LISTENING |
MAIN_EVENT_STOP_LISTENING |
MAIN_EVENT_ACTIVATION_DONE |
MAIN_EVENT_STATE_CHANGED; MAIN_EVENT_STATE_CHANGED;
while (true) { while (true) {
@ -226,7 +243,8 @@ void Application::Run() {
if (bits & MAIN_EVENT_ERROR) { if (bits & MAIN_EVENT_ERROR) {
SetDeviceState(kDeviceStateIdle); SetDeviceState(kDeviceStateIdle);
Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark",
Lang::Sounds::OGG_EXCLAMATION);
} }
if (bits & MAIN_EVENT_NETWORK_CONNECTED) { if (bits & MAIN_EVENT_NETWORK_CONNECTED) {
@ -290,7 +308,7 @@ void Application::Run() {
auto display = Board::GetInstance().GetDisplay(); auto display = Board::GetInstance().GetDisplay();
display->UpdateStatusBar(); display->UpdateStatusBar();
CheckFatigueReminder(); CheckFatigueReminder();
// Print debug info every 10 seconds // Print debug info every 10 seconds
if (clock_ticks_ % 10 == 0) { if (clock_ticks_ % 10 == 0) {
SystemInfo::PrintHeapStats(); SystemInfo::PrintHeapStats();
@ -311,12 +329,14 @@ void Application::HandleNetworkConnectedEvent() {
return; return;
} }
xTaskCreate([](void* arg) { xTaskCreate(
Application* app = static_cast<Application*>(arg); [](void* arg) {
app->ActivationTask(); Application* app = static_cast<Application*>(arg);
app->activation_task_handle_ = nullptr; app->ActivationTask();
vTaskDelete(NULL); app->activation_task_handle_ = nullptr;
}, "activation", 4096 * 2, this, 2, &activation_task_handle_); vTaskDelete(NULL);
},
"activation", 4096 * 2, this, 2, &activation_task_handle_);
} }
// Update the status bar immediately to show the network state // Update the status bar immediately to show the network state
@ -327,7 +347,8 @@ void Application::HandleNetworkConnectedEvent() {
void Application::HandleNetworkDisconnectedEvent() { void Application::HandleNetworkDisconnectedEvent() {
// Close current conversation when network disconnected // Close current conversation when network disconnected
auto state = GetDeviceState(); auto state = GetDeviceState();
if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) { if (state == kDeviceStateConnecting || state == kDeviceStateListening ||
state == kDeviceStateSpeaking) {
ESP_LOGI(TAG, "Closing audio channel due to network disconnection"); ESP_LOGI(TAG, "Closing audio channel due to network disconnection");
protocol_->CloseAudioChannel(); protocol_->CloseAudioChannel();
} }
@ -403,7 +424,7 @@ void Application::CheckAssetsVersion() {
ESP_LOGW(TAG, "Assets partition is disabled for board %s", BOARD_NAME); ESP_LOGW(TAG, "Assets partition is disabled for board %s", BOARD_NAME);
return; return;
} }
Settings settings("assets", true); Settings settings("assets", true);
// Check if there is a new assets need to be downloaded // Check if there is a new assets need to be downloaded
std::string download_url = settings.GetString("download_url"); std::string download_url = settings.GetString("download_url");
@ -413,27 +434,30 @@ void Application::CheckAssetsVersion() {
char message[256]; char message[256];
snprintf(message, sizeof(message), Lang::Strings::FOUND_NEW_ASSETS, download_url.c_str()); snprintf(message, sizeof(message), Lang::Strings::FOUND_NEW_ASSETS, download_url.c_str());
Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down", Lang::Sounds::OGG_UPGRADE); Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down",
Lang::Sounds::OGG_UPGRADE);
// Wait for the audio service to be idle for 3 seconds // Wait for the audio service to be idle for 3 seconds
vTaskDelay(pdMS_TO_TICKS(3000)); vTaskDelay(pdMS_TO_TICKS(3000));
SetDeviceState(kDeviceStateUpgrading); SetDeviceState(kDeviceStateUpgrading);
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
display->SetChatMessage("system", Lang::Strings::PLEASE_WAIT); display->SetChatMessage("system", Lang::Strings::PLEASE_WAIT);
bool success = assets.Download(download_url, [this, display](int progress, size_t speed) -> void { bool success =
char buffer[32]; assets.Download(download_url, [this, display](int progress, size_t speed) -> void {
snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024); char buffer[32];
Schedule([display, message = std::string(buffer)]() { snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024);
display->SetChatMessage("system", message.c_str()); Schedule([display, message = std::string(buffer)]() {
display->SetChatMessage("system", message.c_str());
});
}); });
});
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
vTaskDelay(pdMS_TO_TICKS(1000)); vTaskDelay(pdMS_TO_TICKS(1000));
if (!success) { if (!success) {
Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark",
Lang::Sounds::OGG_EXCLAMATION);
vTaskDelay(pdMS_TO_TICKS(2000)); vTaskDelay(pdMS_TO_TICKS(2000));
SetDeviceState(kDeviceStateActivating); SetDeviceState(kDeviceStateActivating);
return; return;
@ -449,7 +473,7 @@ void Application::CheckAssetsVersion() {
void Application::CheckNewVersion() { void Application::CheckNewVersion() {
const int MAX_RETRY = 10; const int MAX_RETRY = 10;
int retry_count = 0; int retry_count = 0;
int retry_delay = 10; // Initial retry delay in seconds int retry_delay = 10; // Initial retry delay in seconds
auto& board = Board::GetInstance(); auto& board = Board::GetInstance();
while (true) { while (true) {
@ -465,27 +489,30 @@ void Application::CheckNewVersion() {
} }
char error_message[128]; char error_message[128];
snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err, ota_->GetCheckVersionUrl().c_str()); snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err,
ota_->GetCheckVersionUrl().c_str());
char buffer[256]; char buffer[256];
snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay, error_message); snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay,
error_message);
Alert(Lang::Strings::ERROR, buffer, "cloud_slash", Lang::Sounds::OGG_EXCLAMATION); Alert(Lang::Strings::ERROR, buffer, "cloud_slash", Lang::Sounds::OGG_EXCLAMATION);
ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay, retry_count, MAX_RETRY); ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay,
retry_count, MAX_RETRY);
for (int i = 0; i < retry_delay; i++) { for (int i = 0; i < retry_delay; i++) {
vTaskDelay(pdMS_TO_TICKS(1000)); vTaskDelay(pdMS_TO_TICKS(1000));
if (GetDeviceState() == kDeviceStateIdle) { if (GetDeviceState() == kDeviceStateIdle) {
break; break;
} }
} }
retry_delay *= 2; // Double the retry delay retry_delay *= 2; // Double the retry delay
continue; continue;
} }
retry_count = 0; retry_count = 0;
retry_delay = 10; // Reset retry delay retry_delay = 10; // Reset retry delay
if (ota_->HasNewVersion()) { if (ota_->HasNewVersion()) {
if (UpgradeFirmware(ota_->GetFirmwareUrl(), ota_->GetFirmwareVersion())) { if (UpgradeFirmware(ota_->GetFirmwareUrl(), ota_->GetFirmwareVersion())) {
return; // This line will never be reached after reboot return; // This line will never be reached after reboot
} }
// If upgrade failed, continue to normal operation // If upgrade failed, continue to normal operation
} }
@ -540,29 +567,29 @@ void Application::InitializeProtocol() {
protocol_ = std::make_unique<MqttProtocol>(); protocol_ = std::make_unique<MqttProtocol>();
} }
protocol_->OnConnected([this]() { protocol_->OnConnected([this]() { DismissAlert(); });
DismissAlert();
});
protocol_->OnNetworkError([this](const std::string& message) { protocol_->OnNetworkError([this](const std::string& message) {
last_error_message_ = message; last_error_message_ = message;
xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR); xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR);
}); });
protocol_->OnIncomingAudio([this](std::unique_ptr<AudioStreamPacket> packet) { protocol_->OnIncomingAudio([this](std::unique_ptr<AudioStreamPacket> packet) {
if (GetDeviceState() == kDeviceStateSpeaking) { if (GetDeviceState() == kDeviceStateSpeaking) {
audio_service_.PushPacketToDecodeQueue(std::move(packet)); audio_service_.PushPacketToDecodeQueue(std::move(packet));
} }
}); });
protocol_->OnAudioChannelOpened([this, codec, &board]() { protocol_->OnAudioChannelOpened([this, codec, &board]() {
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE); board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
if (protocol_->server_sample_rate() != codec->output_sample_rate()) { if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion", ESP_LOGW(TAG,
protocol_->server_sample_rate(), codec->output_sample_rate()); "Server sample rate %d does not match device output sample rate %d, "
"resampling may cause distortion",
protocol_->server_sample_rate(), codec->output_sample_rate());
} }
}); });
protocol_->OnAudioChannelClosed([this, &board]() { protocol_->OnAudioChannelClosed([this, &board]() {
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
Schedule([this]() { Schedule([this]() {
@ -571,7 +598,7 @@ void Application::InitializeProtocol() {
SetDeviceState(kDeviceStateIdle); SetDeviceState(kDeviceStateIdle);
}); });
}); });
protocol_->OnIncomingJson([this, display](const cJSON* root) { protocol_->OnIncomingJson([this, display](const cJSON* root) {
// Parse JSON data // Parse JSON data
auto type = cJSON_GetObjectItem(root, "type"); auto type = cJSON_GetObjectItem(root, "type");
@ -627,9 +654,7 @@ void Application::InitializeProtocol() {
ESP_LOGI(TAG, "System command: %s", command->valuestring); ESP_LOGI(TAG, "System command: %s", command->valuestring);
if (strcmp(command->valuestring, "reboot") == 0) { if (strcmp(command->valuestring, "reboot") == 0) {
// Do a reboot if user requests a OTA update // Do a reboot if user requests a OTA update
Schedule([this]() { Schedule([this]() { Reboot(); });
Reboot();
});
} else { } else {
ESP_LOGW(TAG, "Unknown system command: %s", command->valuestring); ESP_LOGW(TAG, "Unknown system command: %s", command->valuestring);
} }
@ -639,7 +664,8 @@ void Application::InitializeProtocol() {
auto message = cJSON_GetObjectItem(root, "message"); auto message = cJSON_GetObjectItem(root, "message");
auto emotion = cJSON_GetObjectItem(root, "emotion"); auto emotion = cJSON_GetObjectItem(root, "emotion");
if (cJSON_IsString(status) && cJSON_IsString(message) && cJSON_IsString(emotion)) { if (cJSON_IsString(status) && cJSON_IsString(message) && cJSON_IsString(emotion)) {
Alert(status->valuestring, message->valuestring, emotion->valuestring, Lang::Sounds::OGG_VIBRATION); Alert(status->valuestring, message->valuestring, emotion->valuestring,
Lang::Sounds::OGG_VIBRATION);
} else { } else {
ESP_LOGW(TAG, "Alert command requires status, message and emotion"); ESP_LOGW(TAG, "Alert command requires status, message and emotion");
} }
@ -648,9 +674,10 @@ void Application::InitializeProtocol() {
auto payload = cJSON_GetObjectItem(root, "payload"); auto payload = cJSON_GetObjectItem(root, "payload");
ESP_LOGI(TAG, "Received custom message: %s", cJSON_PrintUnformatted(root)); ESP_LOGI(TAG, "Received custom message: %s", cJSON_PrintUnformatted(root));
if (cJSON_IsObject(payload)) { if (cJSON_IsObject(payload)) {
Schedule([this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() { Schedule(
display->SetChatMessage("system", payload_str.c_str()); [this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() {
}); display->SetChatMessage("system", payload_str.c_str());
});
} else { } else {
ESP_LOGW(TAG, "Invalid custom message format: missing payload"); ESP_LOGW(TAG, "Invalid custom message format: missing payload");
} }
@ -659,7 +686,7 @@ void Application::InitializeProtocol() {
ESP_LOGW(TAG, "Unknown message type: %s", type->valuestring); ESP_LOGW(TAG, "Unknown message type: %s", type->valuestring);
} }
}); });
protocol_->Start(); protocol_->Start();
} }
@ -668,32 +695,27 @@ void Application::ShowActivationCode(const std::string& code, const std::string&
char digit; char digit;
const std::string_view& sound; const std::string_view& sound;
}; };
static const std::array<digit_sound, 10> digit_sounds{{ static const std::array<digit_sound, 10> digit_sounds{
digit_sound{'0', Lang::Sounds::OGG_0}, {digit_sound{'0', Lang::Sounds::OGG_0}, digit_sound{'1', Lang::Sounds::OGG_1},
digit_sound{'1', Lang::Sounds::OGG_1}, digit_sound{'2', Lang::Sounds::OGG_2}, digit_sound{'3', Lang::Sounds::OGG_3},
digit_sound{'2', Lang::Sounds::OGG_2}, digit_sound{'4', Lang::Sounds::OGG_4}, digit_sound{'5', Lang::Sounds::OGG_5},
digit_sound{'3', Lang::Sounds::OGG_3}, digit_sound{'6', Lang::Sounds::OGG_6}, digit_sound{'7', Lang::Sounds::OGG_7},
digit_sound{'4', Lang::Sounds::OGG_4}, digit_sound{'8', Lang::Sounds::OGG_8}, digit_sound{'9', Lang::Sounds::OGG_9}}};
digit_sound{'5', Lang::Sounds::OGG_5},
digit_sound{'6', Lang::Sounds::OGG_6},
digit_sound{'7', Lang::Sounds::OGG_7},
digit_sound{'8', Lang::Sounds::OGG_8},
digit_sound{'9', Lang::Sounds::OGG_9}
}};
// This sentence uses 9KB of SRAM, so we need to wait for it to finish // This sentence uses 9KB of SRAM, so we need to wait for it to finish
Alert(Lang::Strings::ACTIVATION, message.c_str(), "link", Lang::Sounds::OGG_ACTIVATION); Alert(Lang::Strings::ACTIVATION, message.c_str(), "link", Lang::Sounds::OGG_ACTIVATION);
for (const auto& digit : code) { for (const auto& digit : code) {
auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(), auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(),
[digit](const digit_sound& ds) { return ds.digit == digit; }); [digit](const digit_sound& ds) { return ds.digit == digit; });
if (it != digit_sounds.end()) { if (it != digit_sounds.end()) {
audio_service_.PlaySound(it->sound); audio_service_.PlaySound(it->sound);
} }
} }
} }
void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) { void Application::Alert(const char* status, const char* message, const char* emotion,
const std::string_view& sound) {
ESP_LOGW(TAG, "Alert [%s] %s: %s", emotion, status, message); ESP_LOGW(TAG, "Alert [%s] %s: %s", emotion, status, message);
auto display = Board::GetInstance().GetDisplay(); auto display = Board::GetInstance().GetDisplay();
display->SetStatus(status); display->SetStatus(status);
@ -724,7 +746,7 @@ void Application::CheckFatigueReminder() {
} }
Settings settings("fatigue", false); Settings settings("fatigue", false);
if (!settings.GetBool("enabled", true)) { if (!settings.GetBool("enabled", kDefaultFatigueEnabled)) {
return; return;
} }
@ -740,6 +762,19 @@ void Application::CheckFatigueReminder() {
return; return;
} }
if (CheckCameraDrowsiness()) {
last_fatigue_reminder_time_us_ = esp_timer_get_time();
TriggerFatigueReminder();
return;
}
if (!settings.GetBool("silence_enabled", kDefaultFatigueSilenceEnabled)) {
fatigue_idle_seconds_ = 0;
fatigue_silence_seconds_ = 0;
fatigue_reminder_triggered_in_listening_ = false;
return;
}
if (state == kDeviceStateIdle) { if (state == kDeviceStateIdle) {
fatigue_idle_seconds_++; fatigue_idle_seconds_++;
@ -787,14 +822,79 @@ void Application::CheckFatigueReminder() {
TriggerFatigueReminder(); TriggerFatigueReminder();
} }
bool Application::CheckCameraDrowsiness() {
auto state = GetDeviceState();
if (state != kDeviceStateIdle && state != kDeviceStateListening) {
fatigue_camera_closed_samples_ = 0;
return false;
}
Settings settings("fatigue", false);
if (!settings.GetBool("camera_enabled", kDefaultFatigueCameraEnabled)) {
fatigue_camera_closed_samples_ = 0;
return false;
}
int interval_sec = settings.GetInt("camera_interval_sec", kDefaultFatigueCameraIntervalSec);
if (interval_sec < 1) {
interval_sec = 1;
} else if (interval_sec > 60) {
interval_sec = 60;
}
int64_t now_us = esp_timer_get_time();
if (last_fatigue_camera_check_time_us_ != 0 &&
now_us - last_fatigue_camera_check_time_us_ <
static_cast<int64_t>(interval_sec) * 1000000) {
return false;
}
last_fatigue_camera_check_time_us_ = now_us;
auto camera = Board::GetInstance().GetCamera();
if (camera == nullptr) {
return false;
}
bool show_debug_preview =
settings.GetBool("camera_debug_preview", kDefaultFatigueCameraDebugPreview);
CameraDrowsinessResult result;
if (!camera->DetectDrowsiness(result, show_debug_preview)) {
fatigue_camera_closed_samples_ = 0;
return false;
}
ESP_LOGI(TAG, "Camera drowsiness: closed=%d score=%.2f baseline=%.2f",
result.eyes_closed ? 1 : 0, result.eye_openness_score, result.baseline_score);
if (result.eyes_closed) {
fatigue_camera_closed_samples_++;
} else {
fatigue_camera_closed_samples_ = 0;
}
int required_samples =
settings.GetInt("camera_closed_samples", kDefaultFatigueCameraClosedSamples);
if (required_samples < 1) {
required_samples = 1;
} else if (required_samples > 10) {
required_samples = 10;
}
if (fatigue_camera_closed_samples_ >= required_samples) {
fatigue_camera_closed_samples_ = 0;
return true;
}
return false;
}
void Application::TriggerFatigueReminder() { void Application::TriggerFatigueReminder() {
Settings settings("fatigue", false); Settings settings("fatigue", false);
std::string emotion = settings.GetString("emotion", kDefaultFatigueEmotion); std::string emotion = settings.GetString("emotion", kDefaultFatigueEmotion);
std::string message = settings.GetString("message", kDefaultFatigueMessage); std::string message = settings.GetString("message", kDefaultFatigueMessage);
std::string sound_asset = settings.GetString("sound_asset"); std::string sound_asset = settings.GetString("sound_asset");
ESP_LOGW(TAG, "Fatigue reminder triggered: silence=%ds, emotion=%s", ESP_LOGW(TAG, "Fatigue reminder triggered: silence=%ds, emotion=%s", fatigue_silence_seconds_,
fatigue_silence_seconds_, emotion.c_str()); emotion.c_str());
auto display = Board::GetInstance().GetDisplay(); auto display = Board::GetInstance().GetDisplay();
display->SetStatus(kDefaultFatigueStatus); display->SetStatus(kDefaultFatigueStatus);
@ -815,21 +915,15 @@ void Application::TriggerFatigueReminder() {
audio_service_.PlaySound(Lang::Sounds::OGG_POPUP); audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
} }
void Application::ToggleChatState() { void Application::ToggleChatState() { xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT); }
xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT);
}
void Application::StartListening() { void Application::StartListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING); }
xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING);
}
void Application::StopListening() { void Application::StopListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING); }
xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING);
}
void Application::HandleToggleChatEvent() { void Application::HandleToggleChatEvent() {
auto state = GetDeviceState(); auto state = GetDeviceState();
if (state == kDeviceStateActivating) { if (state == kDeviceStateActivating) {
SetDeviceState(kDeviceStateIdle); SetDeviceState(kDeviceStateIdle);
return; return;
@ -853,9 +947,7 @@ void Application::HandleToggleChatEvent() {
if (!protocol_->IsAudioChannelOpened()) { if (!protocol_->IsAudioChannelOpened()) {
SetDeviceState(kDeviceStateConnecting); SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update) // Schedule to let the state change be processed first (UI update)
Schedule([this, mode]() { Schedule([this, mode]() { ContinueOpenAudioChannel(mode); });
ContinueOpenAudioChannel(mode);
});
return; return;
} }
SetListeningMode(mode); SetListeningMode(mode);
@ -883,7 +975,7 @@ void Application::ContinueOpenAudioChannel(ListeningMode mode) {
void Application::HandleStartListeningEvent() { void Application::HandleStartListeningEvent() {
auto state = GetDeviceState(); auto state = GetDeviceState();
if (state == kDeviceStateActivating) { if (state == kDeviceStateActivating) {
SetDeviceState(kDeviceStateIdle); SetDeviceState(kDeviceStateIdle);
return; return;
@ -897,14 +989,12 @@ void Application::HandleStartListeningEvent() {
ESP_LOGE(TAG, "Protocol not initialized"); ESP_LOGE(TAG, "Protocol not initialized");
return; return;
} }
if (state == kDeviceStateIdle) { if (state == kDeviceStateIdle) {
if (!protocol_->IsAudioChannelOpened()) { if (!protocol_->IsAudioChannelOpened()) {
SetDeviceState(kDeviceStateConnecting); SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update) // Schedule to let the state change be processed first (UI update)
Schedule([this]() { Schedule([this]() { ContinueOpenAudioChannel(kListeningModeManualStop); });
ContinueOpenAudioChannel(kListeningModeManualStop);
});
return; return;
} }
SetListeningMode(kListeningModeManualStop); SetListeningMode(kListeningModeManualStop);
@ -916,7 +1006,7 @@ void Application::HandleStartListeningEvent() {
void Application::HandleStopListeningEvent() { void Application::HandleStopListeningEvent() {
auto state = GetDeviceState(); auto state = GetDeviceState();
if (state == kDeviceStateAudioTesting) { if (state == kDeviceStateAudioTesting) {
audio_service_.EnableAudioTesting(false); audio_service_.EnableAudioTesting(false);
SetDeviceState(kDeviceStateWifiConfiguring); SetDeviceState(kDeviceStateWifiConfiguring);
@ -946,9 +1036,7 @@ void Application::HandleWakeWordDetectedEvent() {
SetDeviceState(kDeviceStateConnecting); SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update), // Schedule to let the state change be processed first (UI update),
// then continue with OpenAudioChannel which may block for ~1 second // then continue with OpenAudioChannel which may block for ~1 second
Schedule([this, wake_word]() { Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); });
ContinueWakeWordInvoke(wake_word);
});
return; return;
} }
// Channel already opened, continue directly // Channel already opened, continue directly
@ -956,7 +1044,8 @@ void Application::HandleWakeWordDetectedEvent() {
} else if (state == kDeviceStateSpeaking || state == kDeviceStateListening) { } else if (state == kDeviceStateSpeaking || state == kDeviceStateListening) {
AbortSpeaking(kAbortReasonWakeWordDetected); AbortSpeaking(kAbortReasonWakeWordDetected);
// Clear send queue to avoid sending residues to server // Clear send queue to avoid sending residues to server
while (audio_service_.PopPacketFromSendQueue()); while (audio_service_.PopPacketFromSendQueue())
;
if (state == kDeviceStateListening) { if (state == kDeviceStateListening) {
protocol_->SendStartListening(GetDefaultListeningMode()); protocol_->SendStartListening(GetDefaultListeningMode());
@ -1016,13 +1105,13 @@ void Application::HandleStateChangedEvent() {
auto display = board.GetDisplay(); auto display = board.GetDisplay();
auto led = board.GetLed(); auto led = board.GetLed();
led->OnStateChanged(); led->OnStateChanged();
switch (new_state) { switch (new_state) {
case kDeviceStateUnknown: case kDeviceStateUnknown:
case kDeviceStateIdle: case kDeviceStateIdle:
display->SetStatus(Lang::Strings::STANDBY); display->SetStatus(Lang::Strings::STANDBY);
display->ClearChatMessages(); // Clear messages first display->ClearChatMessages(); // Clear messages first
display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count) display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count)
audio_service_.EnableVoiceProcessing(false); audio_service_.EnableVoiceProcessing(false);
audio_service_.EnableWakeWordDetection(true); audio_service_.EnableWakeWordDetection(true);
break; break;
@ -1038,12 +1127,13 @@ void Application::HandleStateChangedEvent() {
// Make sure the audio processor is running // Make sure the audio processor is running
if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) { if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
// For auto mode, wait for playback queue to be empty before enabling voice processing // For auto mode, wait for playback queue to be empty before enabling voice
// This prevents audio truncation when STOP arrives late due to network jitter // processing This prevents audio truncation when STOP arrives late due to network
// jitter
if (listening_mode_ == kListeningModeAutoStop) { if (listening_mode_ == kListeningModeAutoStop) {
audio_service_.WaitForPlaybackQueueEmpty(); audio_service_.WaitForPlaybackQueueEmpty();
} }
// Send the start listening command // Send the start listening command
protocol_->SendStartListening(listening_mode_); protocol_->SendStartListening(listening_mode_);
audio_service_.EnableVoiceProcessing(true); audio_service_.EnableVoiceProcessing(true);
@ -1056,7 +1146,7 @@ void Application::HandleStateChangedEvent() {
// Disable wake word detection in listening mode // Disable wake word detection in listening mode
audio_service_.EnableWakeWordDetection(false); audio_service_.EnableWakeWordDetection(false);
#endif #endif
// Play popup sound after ResetDecoder (in EnableVoiceProcessing) has been called // Play popup sound after ResetDecoder (in EnableVoiceProcessing) has been called
if (play_popup_on_listening_) { if (play_popup_on_listening_) {
play_popup_on_listening_ = false; play_popup_on_listening_ = false;
@ -1135,7 +1225,8 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver
} }
ESP_LOGI(TAG, "Starting firmware upgrade from URL: %s", upgrade_url.c_str()); ESP_LOGI(TAG, "Starting firmware upgrade from URL: %s", upgrade_url.c_str());
Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download", Lang::Sounds::OGG_UPGRADE); Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download",
Lang::Sounds::OGG_UPGRADE);
vTaskDelay(pdMS_TO_TICKS(3000)); vTaskDelay(pdMS_TO_TICKS(3000));
SetDeviceState(kDeviceStateUpgrading); SetDeviceState(kDeviceStateUpgrading);
@ -1157,17 +1248,19 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver
if (!upgrade_success) { if (!upgrade_success) {
// Upgrade failed, restart audio service and continue running // Upgrade failed, restart audio service and continue running
ESP_LOGE(TAG, "Firmware upgrade failed, restarting audio service and continuing operation..."); ESP_LOGE(TAG,
audio_service_.Start(); // Restart audio service "Firmware upgrade failed, restarting audio service and continuing operation...");
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level audio_service_.Start(); // Restart audio service
Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION); board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level
Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark",
Lang::Sounds::OGG_EXCLAMATION);
vTaskDelay(pdMS_TO_TICKS(3000)); vTaskDelay(pdMS_TO_TICKS(3000));
return false; return false;
} else { } else {
// Upgrade success, reboot immediately // Upgrade success, reboot immediately
ESP_LOGI(TAG, "Firmware upgrade successful, rebooting..."); ESP_LOGI(TAG, "Firmware upgrade successful, rebooting...");
display->SetChatMessage("system", "Upgrade successful, rebooting..."); display->SetChatMessage("system", "Upgrade successful, rebooting...");
vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message
Reboot(); Reboot();
return true; return true;
} }
@ -1179,25 +1272,21 @@ void Application::WakeWordInvoke(const std::string& wake_word) {
} }
auto state = GetDeviceState(); auto state = GetDeviceState();
if (state == kDeviceStateIdle) { if (state == kDeviceStateIdle) {
audio_service_.EncodeWakeWord(); audio_service_.EncodeWakeWord();
if (!protocol_->IsAudioChannelOpened()) { if (!protocol_->IsAudioChannelOpened()) {
SetDeviceState(kDeviceStateConnecting); SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update) // Schedule to let the state change be processed first (UI update)
Schedule([this, wake_word]() { Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); });
ContinueWakeWordInvoke(wake_word);
});
return; return;
} }
// Channel already opened, continue directly // Channel already opened, continue directly
ContinueWakeWordInvoke(wake_word); ContinueWakeWordInvoke(wake_word);
} else if (state == kDeviceStateSpeaking) { } else if (state == kDeviceStateSpeaking) {
Schedule([this]() { Schedule([this]() { AbortSpeaking(kAbortReasonNone); });
AbortSpeaking(kAbortReasonNone); } else if (state == kDeviceStateListening) {
});
} else if (state == kDeviceStateListening) {
Schedule([this]() { Schedule([this]() {
if (protocol_) { if (protocol_) {
protocol_->CloseAudioChannel(); protocol_->CloseAudioChannel();
@ -1238,18 +1327,18 @@ void Application::SetAecMode(AecMode mode) {
auto& board = Board::GetInstance(); auto& board = Board::GetInstance();
auto display = board.GetDisplay(); auto display = board.GetDisplay();
switch (aec_mode_) { switch (aec_mode_) {
case kAecOff: case kAecOff:
audio_service_.EnableDeviceAec(false); audio_service_.EnableDeviceAec(false);
display->ShowNotification(Lang::Strings::RTC_MODE_OFF); display->ShowNotification(Lang::Strings::RTC_MODE_OFF);
break; break;
case kAecOnServerSide: case kAecOnServerSide:
audio_service_.EnableDeviceAec(false); audio_service_.EnableDeviceAec(false);
display->ShowNotification(Lang::Strings::RTC_MODE_ON); display->ShowNotification(Lang::Strings::RTC_MODE_ON);
break; break;
case kAecOnDeviceSide: case kAecOnDeviceSide:
audio_service_.EnableDeviceAec(true); audio_service_.EnableDeviceAec(true);
display->ShowNotification(Lang::Strings::RTC_MODE_ON); display->ShowNotification(Lang::Strings::RTC_MODE_ON);
break; break;
} }
// If the AEC mode is changed, close the audio channel // If the AEC mode is changed, close the audio channel
@ -1259,9 +1348,7 @@ void Application::SetAecMode(AecMode mode) {
}); });
} }
void Application::PlaySound(const std::string_view& sound) { void Application::PlaySound(const std::string_view& sound) { audio_service_.PlaySound(sound); }
audio_service_.PlaySound(sound);
}
void Application::ResetProtocol() { void Application::ResetProtocol() {
Schedule([this]() { Schedule([this]() {

3
main/application.h
View File

@ -143,7 +143,9 @@ private:
bool fatigue_reminder_triggered_in_listening_ = false; bool fatigue_reminder_triggered_in_listening_ = false;
int fatigue_silence_seconds_ = 0; int fatigue_silence_seconds_ = 0;
int fatigue_idle_seconds_ = 0; int fatigue_idle_seconds_ = 0;
int fatigue_camera_closed_samples_ = 0;
int64_t last_fatigue_reminder_time_us_ = 0; int64_t last_fatigue_reminder_time_us_ = 0;
int64_t last_fatigue_camera_check_time_us_ = 0;
int clock_ticks_ = 0; int clock_ticks_ = 0;
TaskHandle_t activation_task_handle_ = nullptr; TaskHandle_t activation_task_handle_ = nullptr;
@ -160,6 +162,7 @@ private:
void ContinueOpenAudioChannel(ListeningMode mode); void ContinueOpenAudioChannel(ListeningMode mode);
void ContinueWakeWordInvoke(const std::string& wake_word); void ContinueWakeWordInvoke(const std::string& wake_word);
void CheckFatigueReminder(); void CheckFatigueReminder();
bool CheckCameraDrowsiness();
void TriggerFatigueReminder(); void TriggerFatigueReminder();
// Activation task (runs in background) // Activation task (runs in background)

8
main/boards/common/camera.h
View File

@ -3,6 +3,13 @@
#include <string> #include <string>
struct CameraDrowsinessResult {
bool valid = false;
bool eyes_closed = false;
float eye_openness_score = 0.0f;
float baseline_score = 0.0f;
};
class Camera { class Camera {
public: public:
virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0; virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0;
@ -10,6 +17,7 @@ public:
virtual bool SetHMirror(bool enabled) = 0; virtual bool SetHMirror(bool enabled) = 0;
virtual bool SetVFlip(bool enabled) = 0; virtual bool SetVFlip(bool enabled) = 0;
virtual bool SetSwapBytes(bool enabled) { return false; } // Optional, default no-op virtual bool SetSwapBytes(bool enabled) { return false; } // Optional, default no-op
virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) { return false; }
virtual std::string Explain(const std::string& question) = 0; virtual std::string Explain(const std::string& question) = 0;
}; };

139
main/boards/common/esp32_camera.cc
View File

@ -5,6 +5,8 @@
#include <cstring> #include <cstring>
#include <esp_log.h> #include <esp_log.h>
#include <img_converters.h> #include <img_converters.h>
#include <algorithm>
#include <cmath>
#include "esp32_camera.h" #include "esp32_camera.h"
#include "board.h" #include "board.h"
@ -17,6 +19,38 @@
#define TAG "Esp32Camera" #define TAG "Esp32Camera"
namespace {
constexpr float kClosedEyeScoreRatio = 0.58f;
constexpr int kEyeSampleStep = 4;
uint8_t Rgb565ToLuma(uint16_t pixel) {
uint8_t r = ((pixel >> 11) & 0x1f) << 3;
uint8_t g = ((pixel >> 5) & 0x3f) << 2;
uint8_t b = (pixel & 0x1f) << 3;
return static_cast<uint8_t>((static_cast<uint16_t>(r) * 30 +
static_cast<uint16_t>(g) * 59 +
static_cast<uint16_t>(b) * 11) / 100);
}
void DrawRectRgb565(uint16_t* pixels, int width, int height, int x0, int y0, int x1, int y1, uint16_t color) {
if (pixels == nullptr || width <= 0 || height <= 0) {
return;
}
x0 = std::clamp(x0, 0, width - 1);
x1 = std::clamp(x1, 0, width - 1);
y0 = std::clamp(y0, 0, height - 1);
y1 = std::clamp(y1, 0, height - 1);
for (int x = x0; x <= x1; x++) {
pixels[y0 * width + x] = color;
pixels[y1 * width + x] = color;
}
for (int y = y0; y <= y1; y++) {
pixels[y * width + x0] = color;
pixels[y * width + x1] = color;
}
}
} // namespace
Esp32Camera::Esp32Camera(const camera_config_t &config) { Esp32Camera::Esp32Camera(const camera_config_t &config) {
esp_err_t err = esp_camera_init(&config); esp_err_t err = esp_camera_init(&config);
if (err != ESP_OK) { if (err != ESP_OK) {
@ -152,6 +186,111 @@ bool Esp32Camera::SetSwapBytes(bool enabled) {
return true; return true;
} }
bool Esp32Camera::DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview) {
result = {};
if (encoder_thread_.joinable()) {
encoder_thread_.join();
}
if (!streaming_on_) {
return false;
}
if (current_fb_) {
esp_camera_fb_return(current_fb_);
current_fb_ = nullptr;
}
camera_fb_t* fb = esp_camera_fb_get();
if (!fb) {
ESP_LOGW(TAG, "Camera drowsiness capture failed");
return false;
}
if (fb->format != PIXFORMAT_RGB565 || fb->width < 80 || fb->height < 80) {
ESP_LOGW(TAG, "Unsupported drowsiness frame: %dx%d format=%d",
fb->width, fb->height, fb->format);
esp_camera_fb_return(fb);
return false;
}
const int width = fb->width;
const int height = fb->height;
const int x0 = width * 22 / 100;
const int x1 = width * 78 / 100;
const int y0 = height * 24 / 100;
const int y1 = height * 46 / 100;
const uint16_t* pixels = reinterpret_cast<const uint16_t*>(fb->buf);
if (show_debug_preview) {
size_t data_size = static_cast<size_t>(width) * static_cast<size_t>(height) * 2;
auto* preview_data = static_cast<uint8_t*>(heap_caps_malloc(data_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
if (preview_data != nullptr) {
auto* dst = reinterpret_cast<uint16_t*>(preview_data);
const auto* src = reinterpret_cast<const uint16_t*>(fb->buf);
size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
for (size_t i = 0; i < pixel_count; i++) {
dst[i] = swap_bytes_enabled_ ? __builtin_bswap16(src[i]) : src[i];
}
DrawRectRgb565(dst, width, height, x0, y0, x1, y1, 0xF800);
auto display = dynamic_cast<LvglDisplay*>(Board::GetInstance().GetDisplay());
if (display != nullptr) {
display->SetPreviewImage(std::make_unique<LvglAllocatedImage>(
preview_data, data_size, width, height, width * 2, LV_COLOR_FORMAT_RGB565));
} else {
heap_caps_free(preview_data);
}
}
}
float vertical_edge_sum = 0.0f;
float horizontal_edge_sum = 0.0f;
int samples = 0;
for (int y = y0; y + kEyeSampleStep < y1; y += kEyeSampleStep) {
for (int x = x0; x + kEyeSampleStep < x1; x += kEyeSampleStep) {
uint16_t p = pixels[y * width + x];
uint16_t px = pixels[y * width + x + kEyeSampleStep];
uint16_t py = pixels[(y + kEyeSampleStep) * width + x];
if (swap_bytes_enabled_) {
p = __builtin_bswap16(p);
px = __builtin_bswap16(px);
py = __builtin_bswap16(py);
}
uint8_t l = Rgb565ToLuma(p);
vertical_edge_sum += std::abs(static_cast<int>(l) - static_cast<int>(Rgb565ToLuma(py)));
horizontal_edge_sum += std::abs(static_cast<int>(l) - static_cast<int>(Rgb565ToLuma(px)));
samples++;
}
}
esp_camera_fb_return(fb);
if (samples == 0) {
return false;
}
// Open eyes usually keep more vertical texture in the fixed eye band.
// This is a lightweight central-face heuristic, not a landmark model.
float score = (vertical_edge_sum + horizontal_edge_sum * 0.35f) / samples;
if (eye_openness_baseline_ <= 0.0f) {
eye_openness_baseline_ = score;
} else if (score > eye_openness_baseline_ * 0.85f) {
eye_openness_baseline_ = eye_openness_baseline_ * 0.90f + score * 0.10f;
} else {
eye_openness_baseline_ = eye_openness_baseline_ * 0.995f + score * 0.005f;
}
result.valid = eye_openness_baseline_ > 1.0f;
result.eye_openness_score = score;
result.baseline_score = eye_openness_baseline_;
result.eyes_closed = result.valid && score < eye_openness_baseline_ * kClosedEyeScoreRatio;
ESP_LOGI(TAG, "Drowsiness frame=%dx%d eye_roi=(%d,%d)-(%d,%d) score=%.2f baseline=%.2f closed=%d",
width, height, x0, y0, x1, y1, result.eye_openness_score, result.baseline_score,
result.eyes_closed ? 1 : 0);
return result.valid;
}
std::string Esp32Camera::Explain(const std::string &question) { std::string Esp32Camera::Explain(const std::string &question) {
if (explain_url_.empty()) { if (explain_url_.empty()) {
throw std::runtime_error("Image explain URL or token is not set"); throw std::runtime_error("Image explain URL or token is not set");

2
main/boards/common/esp32_camera.h
View File

@ -30,6 +30,7 @@ private:
camera_fb_t *current_fb_ = nullptr; camera_fb_t *current_fb_ = nullptr;
uint8_t *encode_buf_ = nullptr; // Buffer for JPEG encoding (with optional byte swap) uint8_t *encode_buf_ = nullptr; // Buffer for JPEG encoding (with optional byte swap)
size_t encode_buf_size_ = 0; size_t encode_buf_size_ = 0;
float eye_openness_baseline_ = 0.0f;
public: public:
Esp32Camera(const camera_config_t &config); Esp32Camera(const camera_config_t &config);
@ -40,5 +41,6 @@ public:
virtual bool SetHMirror(bool enabled) override; virtual bool SetHMirror(bool enabled) override;
virtual bool SetVFlip(bool enabled) override; virtual bool SetVFlip(bool enabled) override;
virtual bool SetSwapBytes(bool enabled) override; virtual bool SetSwapBytes(bool enabled) override;
virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) override;
virtual std::string Explain(const std::string &question) override; virtual std::string Explain(const std::string &question) override;
}; };

7
main/boards/zhengchen-cam/custom-emojis/README.md
View File

@ -29,9 +29,14 @@ Recommended minimum set:
Fatigue reminder: Fatigue reminder:
- Add `wakeup.gif` or `wakeup.png` to make the idle-fatigue reminder show a custom idol animation. - Add `wakeup.gif` or `wakeup.png` to make the idle-fatigue reminder show a custom idol animation.
- The reminder defaults to `wakeup` after 12 seconds of idle time or listening silence, then waits 60 seconds before it can trigger again. - The reminder defaults to `wakeup` when camera drowsiness is detected, then waits 60 seconds before it can trigger again.
- Optional NVS settings in namespace `fatigue`: - Optional NVS settings in namespace `fatigue`:
- `enabled` (`bool`, default `true`) - `enabled` (`bool`, default `true`)
- `camera_enabled` (`bool`, default `true`)
- `camera_debug_preview` (`bool`, default `true`; shows the sampled frame with the eye ROI box)
- `camera_interval_sec` (`int`, default `3`)
- `camera_closed_samples` (`int`, default `2`)
- `silence_enabled` (`bool`, default `false`)
- `idle_timeout_sec` (`int`, default `12`) - `idle_timeout_sec` (`int`, default `12`)
- `listening_timeout_sec` (`int`, default `12`) - `listening_timeout_sec` (`int`, default `12`)
- `cooldown_sec` (`int`, default `60`) - `cooldown_sec` (`int`, default `60`)

123
main/boards/zhengchen-cam/mcp_controller.cc
View File

@ -1,5 +1,6 @@
#include <cJSON.h> #include <cJSON.h>
#include <esp_log.h> #include <esp_log.h>
#include <sdkconfig.h>
#include <cstring> #include <cstring>
@ -13,6 +14,52 @@
#define TAG "MCPController" #define TAG "MCPController"
namespace {
constexpr bool kDefaultFatigueEnabled =
#if defined(CONFIG_FATIGUE_DETECTION_MODE_OFF)
false;
#else
true;
#endif
constexpr bool kDefaultFatigueCameraEnabled =
#if defined(CONFIG_FATIGUE_DETECTION_MODE_CAMERA) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
true;
#else
false;
#endif
constexpr bool kDefaultFatigueSilenceEnabled =
#if defined(CONFIG_FATIGUE_DETECTION_MODE_SILENCE) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
true;
#else
false;
#endif
constexpr bool kDefaultFatigueCameraDebugPreview =
#if defined(CONFIG_FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT)
true;
#else
false;
#endif
std::string GetFatigueDetectionMode(bool enabled, bool camera_enabled, bool silence_enabled) {
if (!enabled) {
return "off";
}
if (camera_enabled && silence_enabled) {
return "both";
}
if (camera_enabled) {
return "camera";
}
if (silence_enabled) {
return "silence";
}
return "off";
}
} // namespace
class MCPController { class MCPController {
public: public:
MCPController() { MCPController() {
@ -42,7 +89,6 @@ public:
app.SetAecMode(kAecOff); app.SetAecMode(kAecOff);
return "{\"success\": true, \"message\": \"AEC对话打断模式已关闭\"}"; return "{\"success\": true, \"message\": \"AEC对话打断模式已关闭\"}";
}else { }else {
auto& board = Board::GetInstance();
app.SetAecMode(kAecOnDeviceSide); app.SetAecMode(kAecOnDeviceSide);
return "{\"success\": true, \"message\": \"AEC对话打断模式已开启\"}"; return "{\"success\": true, \"message\": \"AEC对话打断模式已开启\"}";
@ -79,6 +125,79 @@ public:
} }
); );
mcp_server.AddTool(
"self.fatigue.set_detection_mode",
"切换疲劳提醒检测模式。当用户想切换打瞌睡检测、眨眼检测、不讲话检测、关闭疲劳提醒或查看摄像头取样画面时使用此工具。\n"
"参数：\n"
" `mode`: 检测模式，可选值只有 `camera`(只用摄像头眨眼/闭眼检测)、`silence`(只用不讲话/静默检测)、`both`(两个都开)、`off`(关闭疲劳提醒)\n"
" `debug_preview`: 是否显示摄像头调试预览。默认 false；只有用户明确想看摄像头拍到什么时才设为 true\n"
"返回值：\n"
" 反馈当前疲劳检测模式，不需要确认，立即播报相关数据\n",
PropertyList({
Property("mode", kPropertyTypeString),
Property("debug_preview", kPropertyTypeBoolean, false),
}),
[](const PropertyList& properties) -> ReturnValue {
auto mode = properties["mode"].value<std::string>();
bool debug_preview = properties["debug_preview"].value<bool>();
bool enabled = true;
bool camera_enabled = false;
bool silence_enabled = false;
std::string message;
if (mode == "camera") {
camera_enabled = true;
message = "已切换为摄像头眨眼检测模式";
} else if (mode == "silence") {
silence_enabled = true;
message = "已切换为不讲话静默检测模式";
} else if (mode == "both") {
camera_enabled = true;
silence_enabled = true;
message = "已切换为摄像头加静默双检测模式";
} else if (mode == "off") {
enabled = false;
message = "疲劳提醒检测已关闭";
} else {
return "{\"success\": false, \"message\": \"检测模式无效，只能使用 camera、silence、both 或 off\"}";
}
Settings settings("fatigue", true);
settings.SetBool("enabled", enabled);
settings.SetBool("camera_enabled", camera_enabled);
settings.SetBool("silence_enabled", silence_enabled);
settings.SetBool("camera_debug_preview", debug_preview);
return std::string("{\"success\": true, \"mode\": \"") + mode +
"\", \"debug_preview\": " + (debug_preview ? "true" : "false") +
", \"message\": \"" + message + "\"}";
}
);
mcp_server.AddTool(
"self.fatigue.get_detection_mode",
"获取当前疲劳提醒检测模式。当用户询问现在使用哪种打瞌睡检测模式时使用此工具。\n"
"返回值：\n"
" 当前检测模式和摄像头预览状态\n",
PropertyList(),
[](const PropertyList&) -> ReturnValue {
Settings settings("fatigue", false);
bool enabled = settings.GetBool("enabled", kDefaultFatigueEnabled);
bool camera_enabled = settings.GetBool("camera_enabled", kDefaultFatigueCameraEnabled);
bool silence_enabled =
settings.GetBool("silence_enabled", kDefaultFatigueSilenceEnabled);
bool debug_preview =
settings.GetBool("camera_debug_preview", kDefaultFatigueCameraDebugPreview);
auto mode = GetFatigueDetectionMode(enabled, camera_enabled, silence_enabled);
return std::string("{\"success\": true, \"mode\": \"") + mode +
"\", \"camera_enabled\": " + (camera_enabled ? "true" : "false") +
", \"silence_enabled\": " + (silence_enabled ? "true" : "false") +
", \"debug_preview\": " + (debug_preview ? "true" : "false") + "}";
}
);
ESP_LOGI(TAG, "MCP工具注册完成"); ESP_LOGI(TAG, "MCP工具注册完成");
} }
@ -91,4 +210,4 @@ void InitializeMCPController() {
g_mcp_controller = new MCPController(); g_mcp_controller = new MCPController();
ESP_LOGI(TAG, "注册MCP工具"); ESP_LOGI(TAG, "注册MCP工具");
} }
} }

8
main/display/lcd_display.cc
View File

@ -1206,6 +1206,14 @@ void LcdDisplay::SetEmotion(const char* emotion) {
DisplayLockGuard lock(this); DisplayLockGuard lock(this);
bool use_full_screen_center = strcmp(emotion, "neutral") == 0; bool use_full_screen_center = strcmp(emotion, "neutral") == 0;
if (use_full_screen_center && preview_image_ != nullptr) {
esp_timer_stop(preview_timer_);
if (emoji_box_ != nullptr) {
lv_obj_remove_flag(emoji_box_, LV_OBJ_FLAG_HIDDEN);
}
lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
preview_image_cached_.reset();
}
lv_coord_t top_reserved = use_full_screen_center ? 0 : ObjectHeight(top_bar_); lv_coord_t top_reserved = use_full_screen_center ? 0 : ObjectHeight(top_bar_);
lv_coord_t bottom_reserved = use_full_screen_center ? 0 : ObjectHeight(status_bar_) + ObjectHeight(bottom_bar_); lv_coord_t bottom_reserved = use_full_screen_center ? 0 : ObjectHeight(status_bar_) + ObjectHeight(bottom_bar_);
auto apply_emoji_layout = [this, top_reserved, bottom_reserved](const lv_image_dsc_t* image_dsc) { auto apply_emoji_layout = [this, top_reserved, bottom_reserved](const lv_image_dsc_t* image_dsc) {