verachen/kedia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: cam and slience detect sleep

Browse Source
This commit is contained in:
0Xiao0
2026-06-17 15:02:16 +08:00
parent 154ce461d7
commit a8c6c62c92
9 changed files with 564 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
main/boards/common/camera.h
View File

@ -3,6 +3,13 @@
#include <string>
struct CameraDrowsinessResult {
bool valid = false;
bool eyes_closed = false;
float eye_openness_score = 0.0f;
float baseline_score = 0.0f;
};
class Camera {
public:
virtual void SetExplainUrl(const std::string& url, const std::string& token) = 0;
@ -10,6 +17,7 @@ public:
virtual bool SetHMirror(bool enabled) = 0;
virtual bool SetVFlip(bool enabled) = 0;
virtual bool SetSwapBytes(bool enabled) { return false; } // Optional, default no-op
virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) { return false; }
virtual std::string Explain(const std::string& question) = 0;
};

139
main/boards/common/esp32_camera.cc
View File

@ -5,6 +5,8 @@
#include <cstring>
#include <esp_log.h>
#include <img_converters.h>
#include <algorithm>
#include <cmath>
#include "esp32_camera.h"
#include "board.h"
@ -17,6 +19,38 @@
#define TAG "Esp32Camera"
namespace {
constexpr float kClosedEyeScoreRatio = 0.58f;
constexpr int kEyeSampleStep = 4;
uint8_t Rgb565ToLuma(uint16_t pixel) {
uint8_t r = ((pixel >> 11) & 0x1f) << 3;
uint8_t g = ((pixel >> 5) & 0x3f) << 2;
uint8_t b = (pixel & 0x1f) << 3;
return static_cast<uint8_t>((static_cast<uint16_t>(r) * 30 +
static_cast<uint16_t>(g) * 59 +
static_cast<uint16_t>(b) * 11) / 100);
}
void DrawRectRgb565(uint16_t* pixels, int width, int height, int x0, int y0, int x1, int y1, uint16_t color) {
if (pixels == nullptr || width <= 0 || height <= 0) {
return;
}
x0 = std::clamp(x0, 0, width - 1);
x1 = std::clamp(x1, 0, width - 1);
y0 = std::clamp(y0, 0, height - 1);
y1 = std::clamp(y1, 0, height - 1);
for (int x = x0; x <= x1; x++) {
pixels[y0 * width + x] = color;
pixels[y1 * width + x] = color;
}
for (int y = y0; y <= y1; y++) {
pixels[y * width + x0] = color;
pixels[y * width + x1] = color;
}
}
} // namespace
Esp32Camera::Esp32Camera(const camera_config_t &config) {
esp_err_t err = esp_camera_init(&config);
if (err != ESP_OK) {
@ -152,6 +186,111 @@ bool Esp32Camera::SetSwapBytes(bool enabled) {
return true;
}
bool Esp32Camera::DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview) {
result = {};
if (encoder_thread_.joinable()) {
encoder_thread_.join();
}
if (!streaming_on_) {
return false;
}
if (current_fb_) {
esp_camera_fb_return(current_fb_);
current_fb_ = nullptr;
}
camera_fb_t* fb = esp_camera_fb_get();
if (!fb) {
ESP_LOGW(TAG, "Camera drowsiness capture failed");
return false;
}
if (fb->format != PIXFORMAT_RGB565 || fb->width < 80 || fb->height < 80) {
ESP_LOGW(TAG, "Unsupported drowsiness frame: %dx%d format=%d",
fb->width, fb->height, fb->format);
esp_camera_fb_return(fb);
return false;
}
const int width = fb->width;
const int height = fb->height;
const int x0 = width * 22 / 100;
const int x1 = width * 78 / 100;
const int y0 = height * 24 / 100;
const int y1 = height * 46 / 100;
const uint16_t* pixels = reinterpret_cast<const uint16_t*>(fb->buf);
if (show_debug_preview) {
size_t data_size = static_cast<size_t>(width) * static_cast<size_t>(height) * 2;
auto* preview_data = static_cast<uint8_t*>(heap_caps_malloc(data_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
if (preview_data != nullptr) {
auto* dst = reinterpret_cast<uint16_t*>(preview_data);
const auto* src = reinterpret_cast<const uint16_t*>(fb->buf);
size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
for (size_t i = 0; i < pixel_count; i++) {
dst[i] = swap_bytes_enabled_ ? __builtin_bswap16(src[i]) : src[i];
}
DrawRectRgb565(dst, width, height, x0, y0, x1, y1, 0xF800);
auto display = dynamic_cast<LvglDisplay*>(Board::GetInstance().GetDisplay());
if (display != nullptr) {
display->SetPreviewImage(std::make_unique<LvglAllocatedImage>(
preview_data, data_size, width, height, width * 2, LV_COLOR_FORMAT_RGB565));
} else {
heap_caps_free(preview_data);
}
}
}
float vertical_edge_sum = 0.0f;
float horizontal_edge_sum = 0.0f;
int samples = 0;
for (int y = y0; y + kEyeSampleStep < y1; y += kEyeSampleStep) {
for (int x = x0; x + kEyeSampleStep < x1; x += kEyeSampleStep) {
uint16_t p = pixels[y * width + x];
uint16_t px = pixels[y * width + x + kEyeSampleStep];
uint16_t py = pixels[(y + kEyeSampleStep) * width + x];
if (swap_bytes_enabled_) {
p = __builtin_bswap16(p);
px = __builtin_bswap16(px);
py = __builtin_bswap16(py);
}
uint8_t l = Rgb565ToLuma(p);
vertical_edge_sum += std::abs(static_cast<int>(l) - static_cast<int>(Rgb565ToLuma(py)));
horizontal_edge_sum += std::abs(static_cast<int>(l) - static_cast<int>(Rgb565ToLuma(px)));
samples++;
}
}
esp_camera_fb_return(fb);
if (samples == 0) {
return false;
}
// Open eyes usually keep more vertical texture in the fixed eye band.
// This is a lightweight central-face heuristic, not a landmark model.
float score = (vertical_edge_sum + horizontal_edge_sum * 0.35f) / samples;
if (eye_openness_baseline_ <= 0.0f) {
eye_openness_baseline_ = score;
} else if (score > eye_openness_baseline_ * 0.85f) {
eye_openness_baseline_ = eye_openness_baseline_ * 0.90f + score * 0.10f;
} else {
eye_openness_baseline_ = eye_openness_baseline_ * 0.995f + score * 0.005f;
}
result.valid = eye_openness_baseline_ > 1.0f;
result.eye_openness_score = score;
result.baseline_score = eye_openness_baseline_;
result.eyes_closed = result.valid && score < eye_openness_baseline_ * kClosedEyeScoreRatio;
ESP_LOGI(TAG, "Drowsiness frame=%dx%d eye_roi=(%d,%d)-(%d,%d) score=%.2f baseline=%.2f closed=%d",
width, height, x0, y0, x1, y1, result.eye_openness_score, result.baseline_score,
result.eyes_closed ? 1 : 0);
return result.valid;
}
std::string Esp32Camera::Explain(const std::string &question) {
if (explain_url_.empty()) {
throw std::runtime_error("Image explain URL or token is not set");

2
main/boards/common/esp32_camera.h
View File

@ -30,6 +30,7 @@ private:
camera_fb_t *current_fb_ = nullptr;
uint8_t *encode_buf_ = nullptr; // Buffer for JPEG encoding (with optional byte swap)
size_t encode_buf_size_ = 0;
float eye_openness_baseline_ = 0.0f;
public:
Esp32Camera(const camera_config_t &config);
@ -40,5 +41,6 @@ public:
virtual bool SetHMirror(bool enabled) override;
virtual bool SetVFlip(bool enabled) override;
virtual bool SetSwapBytes(bool enabled) override;
virtual bool DetectDrowsiness(CameraDrowsinessResult& result, bool show_debug_preview = false) override;
virtual std::string Explain(const std::string &question) override;
};

7
main/boards/zhengchen-cam/custom-emojis/README.md
View File

@ -29,9 +29,14 @@ Recommended minimum set:
Fatigue reminder:
- Add `wakeup.gif` or `wakeup.png` to make the idle-fatigue reminder show a custom idol animation.
- The reminder defaults to `wakeup` after 12 seconds of idle time or listening silence, then waits 60 seconds before it can trigger again.
- The reminder defaults to `wakeup` when camera drowsiness is detected, then waits 60 seconds before it can trigger again.
- Optional NVS settings in namespace `fatigue`:
- `enabled` (`bool`, default `true`)
- `camera_enabled` (`bool`, default `true`)
- `camera_debug_preview` (`bool`, default `true`; shows the sampled frame with the eye ROI box)
- `camera_interval_sec` (`int`, default `3`)
- `camera_closed_samples` (`int`, default `2`)
- `silence_enabled` (`bool`, default `false`)
- `idle_timeout_sec` (`int`, default `12`)
- `listening_timeout_sec` (`int`, default `12`)
- `cooldown_sec` (`int`, default `60`)

123
main/boards/zhengchen-cam/mcp_controller.cc
View File

@ -1,5 +1,6 @@
#include <cJSON.h>
#include <esp_log.h>
#include <sdkconfig.h>
#include <cstring>
@ -13,6 +14,52 @@
#define TAG "MCPController"
namespace {
constexpr bool kDefaultFatigueEnabled =
#if defined(CONFIG_FATIGUE_DETECTION_MODE_OFF)
false;
#else
true;
#endif
constexpr bool kDefaultFatigueCameraEnabled =
#if defined(CONFIG_FATIGUE_DETECTION_MODE_CAMERA) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
true;
#else
false;
#endif
constexpr bool kDefaultFatigueSilenceEnabled =
#if defined(CONFIG_FATIGUE_DETECTION_MODE_SILENCE) || defined(CONFIG_FATIGUE_DETECTION_MODE_BOTH)
true;
#else
false;
#endif
constexpr bool kDefaultFatigueCameraDebugPreview =
#if defined(CONFIG_FATIGUE_CAMERA_DEBUG_PREVIEW_DEFAULT)
true;
#else
false;
#endif
std::string GetFatigueDetectionMode(bool enabled, bool camera_enabled, bool silence_enabled) {
if (!enabled) {
return "off";
}
if (camera_enabled && silence_enabled) {
return "both";
}
if (camera_enabled) {
return "camera";
}
if (silence_enabled) {
return "silence";
}
return "off";
}
} // namespace
class MCPController {
public:
MCPController() {
@ -42,7 +89,6 @@ public:
app.SetAecMode(kAecOff);
return "{\"success\": true, \"message\": \"AEC对话打断模式已关闭\"}";
}else {
auto& board = Board::GetInstance();
app.SetAecMode(kAecOnDeviceSide);
return "{\"success\": true, \"message\": \"AEC对话打断模式已开启\"}";
@ -79,6 +125,79 @@ public:
}
);
mcp_server.AddTool(
"self.fatigue.set_detection_mode",
"切换疲劳提醒检测模式。当用户想切换打瞌睡检测、眨眼检测、不讲话检测、关闭疲劳提醒或查看摄像头取样画面时使用此工具。\n"
"参数：\n"
" `mode`: 检测模式，可选值只有 `camera`(只用摄像头眨眼/闭眼检测)、`silence`(只用不讲话/静默检测)、`both`(两个都开)、`off`(关闭疲劳提醒)\n"
" `debug_preview`: 是否显示摄像头调试预览。默认 false；只有用户明确想看摄像头拍到什么时才设为 true\n"
"返回值：\n"
" 反馈当前疲劳检测模式，不需要确认，立即播报相关数据\n",
PropertyList({
Property("mode", kPropertyTypeString),
Property("debug_preview", kPropertyTypeBoolean, false),
}),
[](const PropertyList& properties) -> ReturnValue {
auto mode = properties["mode"].value<std::string>();
bool debug_preview = properties["debug_preview"].value<bool>();
bool enabled = true;
bool camera_enabled = false;
bool silence_enabled = false;
std::string message;
if (mode == "camera") {
camera_enabled = true;
message = "已切换为摄像头眨眼检测模式";
} else if (mode == "silence") {
silence_enabled = true;
message = "已切换为不讲话静默检测模式";
} else if (mode == "both") {
camera_enabled = true;
silence_enabled = true;
message = "已切换为摄像头加静默双检测模式";
} else if (mode == "off") {
enabled = false;
message = "疲劳提醒检测已关闭";
} else {
return "{\"success\": false, \"message\": \"检测模式无效，只能使用 camera、silence、both 或 off\"}";
}
Settings settings("fatigue", true);
settings.SetBool("enabled", enabled);
settings.SetBool("camera_enabled", camera_enabled);
settings.SetBool("silence_enabled", silence_enabled);
settings.SetBool("camera_debug_preview", debug_preview);
return std::string("{\"success\": true, \"mode\": \"") + mode +
"\", \"debug_preview\": " + (debug_preview ? "true" : "false") +
", \"message\": \"" + message + "\"}";
}
);
mcp_server.AddTool(
"self.fatigue.get_detection_mode",
"获取当前疲劳提醒检测模式。当用户询问现在使用哪种打瞌睡检测模式时使用此工具。\n"
"返回值：\n"
" 当前检测模式和摄像头预览状态\n",
PropertyList(),
[](const PropertyList&) -> ReturnValue {
Settings settings("fatigue", false);
bool enabled = settings.GetBool("enabled", kDefaultFatigueEnabled);
bool camera_enabled = settings.GetBool("camera_enabled", kDefaultFatigueCameraEnabled);
bool silence_enabled =
settings.GetBool("silence_enabled", kDefaultFatigueSilenceEnabled);
bool debug_preview =
settings.GetBool("camera_debug_preview", kDefaultFatigueCameraDebugPreview);
auto mode = GetFatigueDetectionMode(enabled, camera_enabled, silence_enabled);
return std::string("{\"success\": true, \"mode\": \"") + mode +
"\", \"camera_enabled\": " + (camera_enabled ? "true" : "false") +
", \"silence_enabled\": " + (silence_enabled ? "true" : "false") +
", \"debug_preview\": " + (debug_preview ? "true" : "false") + "}";
}
);
ESP_LOGI(TAG, "MCP工具注册完成");
}
@ -91,4 +210,4 @@ void InitializeMCPController() {
g_mcp_controller = new MCPController();
ESP_LOGI(TAG, "注册MCP工具");
}
}
}