verachen/xiaozhi-esp32
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: voice interupt

Browse Source
This commit is contained in:
0Xiao0
2026-06-12 11:38:47 +08:00
parent 9637e09aef
commit 2c4329fd84
22 changed files with 816 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

348
main/application.cc
View File

@ -1,25 +1,24 @@
#include "application.h"
#include "assets.h"
#include "assets/lang_config.h"
#include "audio_codec.h"
#include "board.h"
#include "display.h"
#include "system_info.h"
#include "audio_codec.h"
#include "mqtt_protocol.h"
#include "websocket_protocol.h"
#include "assets/lang_config.h"
#include "mcp_server.h"
#include "assets.h"
#include "mqtt_protocol.h"
#include "settings.h"
#include "system_info.h"
#include "websocket_protocol.h"
#include <cstring>
#include <esp_log.h>
#include <cJSON.h>
#include <driver/gpio.h>
#include <esp_log.h>
#include <arpa/inet.h>
#include <cJSON.h>
#include <font_awesome.h>
#include <cstring>
#define TAG "Application"
Application::Application() {
event_group_ = xEventGroupCreate();
@ -33,16 +32,16 @@ Application::Application() {
aec_mode_ = kAecOff;
#endif
esp_timer_create_args_t clock_timer_args = {
.callback = [](void* arg) {
Application* app = (Application*)arg;
xEventGroupSetBits(app->event_group_, MAIN_EVENT_CLOCK_TICK);
},
.arg = this,
.dispatch_method = ESP_TIMER_TASK,
.name = "clock_timer",
.skip_unhandled_events = true
};
esp_timer_create_args_t clock_timer_args = {.callback =
[](void* arg) {
Application* app = (Application*)arg;
xEventGroupSetBits(app->event_group_,
MAIN_EVENT_CLOCK_TICK);
},
.arg = this,
.dispatch_method = ESP_TIMER_TASK,
.name = "clock_timer",
.skip_unhandled_events = true};
esp_timer_create(&clock_timer_args, &clock_timer_handle_);
}
@ -54,9 +53,7 @@ Application::~Application() {
vEventGroupDelete(event_group_);
}
bool Application::SetDeviceState(DeviceState state) {
return state_machine_.TransitionTo(state);
}
bool Application::SetDeviceState(DeviceState state) { return state_machine_.TransitionTo(state); }
void Application::Initialize() {
auto& board = Board::GetInstance();
@ -102,7 +99,7 @@ void Application::Initialize() {
// Set network event callback for UI updates and network state handling
board.SetNetworkEventCallback([this](NetworkEvent event, const std::string& data) {
auto display = Board::GetInstance().GetDisplay();
switch (event) {
case NetworkEvent::Scanning:
display->ShowNotification(Lang::Strings::SCANNING_WIFI, 30000);
@ -142,13 +139,16 @@ void Application::Initialize() {
display->SetStatus(Lang::Strings::DETECTING_MODULE);
break;
case NetworkEvent::ModemErrorNoSim:
Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_PIN);
Alert(Lang::Strings::ERROR, Lang::Strings::PIN_ERROR, "triangle_exclamation",
Lang::Sounds::OGG_ERR_PIN);
break;
case NetworkEvent::ModemErrorRegDenied:
Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation", Lang::Sounds::OGG_ERR_REG);
Alert(Lang::Strings::ERROR, Lang::Strings::REG_ERROR, "triangle_exclamation",
Lang::Sounds::OGG_ERR_REG);
break;
case NetworkEvent::ModemErrorInitFailed:
Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation", Lang::Sounds::OGG_EXCLAMATION);
Alert(Lang::Strings::ERROR, Lang::Strings::MODEM_INIT_ERROR, "triangle_exclamation",
Lang::Sounds::OGG_EXCLAMATION);
break;
case NetworkEvent::ModemErrorTimeout:
display->SetStatus(Lang::Strings::REGISTERING_NETWORK);
@ -167,19 +167,11 @@ void Application::Run() {
// Set the priority of the main task to 10
vTaskPrioritySet(nullptr, 10);
const EventBits_t ALL_EVENTS =
MAIN_EVENT_SCHEDULE |
MAIN_EVENT_SEND_AUDIO |
MAIN_EVENT_WAKE_WORD_DETECTED |
MAIN_EVENT_VAD_CHANGE |
MAIN_EVENT_CLOCK_TICK |
MAIN_EVENT_ERROR |
MAIN_EVENT_NETWORK_CONNECTED |
MAIN_EVENT_NETWORK_DISCONNECTED |
MAIN_EVENT_TOGGLE_CHAT |
MAIN_EVENT_START_LISTENING |
MAIN_EVENT_STOP_LISTENING |
MAIN_EVENT_ACTIVATION_DONE |
const EventBits_t ALL_EVENTS =
MAIN_EVENT_SCHEDULE | MAIN_EVENT_SEND_AUDIO | MAIN_EVENT_WAKE_WORD_DETECTED |
MAIN_EVENT_VAD_CHANGE | MAIN_EVENT_CLOCK_TICK | MAIN_EVENT_ERROR |
MAIN_EVENT_NETWORK_CONNECTED | MAIN_EVENT_NETWORK_DISCONNECTED | MAIN_EVENT_TOGGLE_CHAT |
MAIN_EVENT_START_LISTENING | MAIN_EVENT_STOP_LISTENING | MAIN_EVENT_ACTIVATION_DONE |
MAIN_EVENT_STATE_CHANGED;
while (true) {
@ -187,7 +179,8 @@ void Application::Run() {
if (bits & MAIN_EVENT_ERROR) {
SetDeviceState(kDeviceStateIdle);
Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
Alert(Lang::Strings::ERROR, last_error_message_.c_str(), "circle_xmark",
Lang::Sounds::OGG_EXCLAMATION);
}
if (bits & MAIN_EVENT_NETWORK_CONNECTED) {
@ -257,7 +250,7 @@ void Application::Run() {
clock_ticks_++;
auto display = Board::GetInstance().GetDisplay();
display->UpdateStatusBar();
// Print debug info every 10 seconds
if (clock_ticks_ % 10 == 0) {
SystemInfo::PrintHeapStats();
@ -278,12 +271,14 @@ void Application::HandleNetworkConnectedEvent() {
return;
}
xTaskCreate([](void* arg) {
Application* app = static_cast<Application*>(arg);
app->ActivationTask();
app->activation_task_handle_ = nullptr;
vTaskDelete(NULL);
}, "activation", 4096 * 2, this, 2, &activation_task_handle_);
xTaskCreate(
[](void* arg) {
Application* app = static_cast<Application*>(arg);
app->ActivationTask();
app->activation_task_handle_ = nullptr;
vTaskDelete(NULL);
},
"activation", 4096 * 2, this, 2, &activation_task_handle_);
}
// Update the status bar immediately to show the network state
@ -294,7 +289,8 @@ void Application::HandleNetworkConnectedEvent() {
void Application::HandleNetworkDisconnectedEvent() {
// Close current conversation when network disconnected
auto state = GetDeviceState();
if (state == kDeviceStateConnecting || state == kDeviceStateListening || state == kDeviceStateSpeaking) {
if (state == kDeviceStateConnecting || state == kDeviceStateListening ||
state == kDeviceStateThinking || state == kDeviceStateSpeaking) {
ESP_LOGI(TAG, "Closing audio channel due to network disconnection");
protocol_->CloseAudioChannel();
}
@ -369,7 +365,7 @@ void Application::CheckAssetsVersion() {
ESP_LOGW(TAG, "Assets partition is disabled for board %s", BOARD_NAME);
return;
}
Settings settings("assets", true);
// Check if there is a new assets need to be downloaded
std::string download_url = settings.GetString("download_url");
@ -379,27 +375,30 @@ void Application::CheckAssetsVersion() {
char message[256];
snprintf(message, sizeof(message), Lang::Strings::FOUND_NEW_ASSETS, download_url.c_str());
Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down", Lang::Sounds::OGG_UPGRADE);
Alert(Lang::Strings::LOADING_ASSETS, message, "cloud_arrow_down",
Lang::Sounds::OGG_UPGRADE);
// Wait for the audio service to be idle for 3 seconds
vTaskDelay(pdMS_TO_TICKS(3000));
SetDeviceState(kDeviceStateUpgrading);
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
display->SetChatMessage("system", Lang::Strings::PLEASE_WAIT);
bool success = assets.Download(download_url, [this, display](int progress, size_t speed) -> void {
char buffer[32];
snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024);
Schedule([display, message = std::string(buffer)]() {
display->SetChatMessage("system", message.c_str());
bool success =
assets.Download(download_url, [this, display](int progress, size_t speed) -> void {
char buffer[32];
snprintf(buffer, sizeof(buffer), "%d%% %uKB/s", progress, speed / 1024);
Schedule([display, message = std::string(buffer)]() {
display->SetChatMessage("system", message.c_str());
});
});
});
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
vTaskDelay(pdMS_TO_TICKS(1000));
if (!success) {
Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
Alert(Lang::Strings::ERROR, Lang::Strings::DOWNLOAD_ASSETS_FAILED, "circle_xmark",
Lang::Sounds::OGG_EXCLAMATION);
vTaskDelay(pdMS_TO_TICKS(2000));
SetDeviceState(kDeviceStateActivating);
return;
@ -415,7 +414,7 @@ void Application::CheckAssetsVersion() {
void Application::CheckNewVersion() {
const int MAX_RETRY = 10;
int retry_count = 0;
int retry_delay = 10; // Initial retry delay in seconds
int retry_delay = 10; // Initial retry delay in seconds
auto& board = Board::GetInstance();
while (true) {
@ -431,27 +430,30 @@ void Application::CheckNewVersion() {
}
char error_message[128];
snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err, ota_->GetCheckVersionUrl().c_str());
snprintf(error_message, sizeof(error_message), "code=%d, url=%s", err,
ota_->GetCheckVersionUrl().c_str());
char buffer[256];
snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay, error_message);
snprintf(buffer, sizeof(buffer), Lang::Strings::CHECK_NEW_VERSION_FAILED, retry_delay,
error_message);
Alert(Lang::Strings::ERROR, buffer, "cloud_slash", Lang::Sounds::OGG_EXCLAMATION);
ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay, retry_count, MAX_RETRY);
ESP_LOGW(TAG, "Check new version failed, retry in %d seconds (%d/%d)", retry_delay,
retry_count, MAX_RETRY);
for (int i = 0; i < retry_delay; i++) {
vTaskDelay(pdMS_TO_TICKS(1000));
if (GetDeviceState() == kDeviceStateIdle) {
break;
}
}
retry_delay *= 2; // Double the retry delay
retry_delay *= 2; // Double the retry delay
continue;
}
retry_count = 0;
retry_delay = 10; // Reset retry delay
retry_delay = 10; // Reset retry delay
if (ota_->HasNewVersion()) {
if (UpgradeFirmware(ota_->GetFirmwareUrl(), ota_->GetFirmwareVersion())) {
return; // This line will never be reached after reboot
return; // This line will never be reached after reboot
}
// If upgrade failed, continue to normal operation
}
@ -507,31 +509,32 @@ void Application::InitializeProtocol() {
}
#endif
protocol_->OnConnected([this]() {
DismissAlert();
});
protocol_->OnConnected([this]() { DismissAlert(); });
protocol_->OnNetworkError([this](const std::string& message) {
last_error_message_ = message;
xEventGroupSetBits(event_group_, MAIN_EVENT_ERROR);
});
protocol_->OnIncomingAudio([this](std::unique_ptr<AudioStreamPacket> packet) {
if (GetDeviceState() == kDeviceStateSpeaking) {
if (accepting_tts_audio_.load() || GetDeviceState() == kDeviceStateSpeaking) {
audio_service_.PushPacketToDecodeQueue(std::move(packet));
}
});
protocol_->OnAudioChannelOpened([this, codec, &board]() {
board.SetPowerSaveLevel(PowerSaveLevel::PERFORMANCE);
if (protocol_->server_sample_rate() != codec->output_sample_rate()) {
ESP_LOGW(TAG, "Server sample rate %d does not match device output sample rate %d, resampling may cause distortion",
protocol_->server_sample_rate(), codec->output_sample_rate());
ESP_LOGW(TAG,
"Server sample rate %d does not match device output sample rate %d, "
"resampling may cause distortion",
protocol_->server_sample_rate(), codec->output_sample_rate());
}
});
protocol_->OnAudioChannelClosed([this, &board]() {
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER);
accepting_tts_audio_.store(false);
Schedule([this]() {
if (GetDeviceState() == kDeviceStateConnecting) {
return;
@ -541,20 +544,26 @@ void Application::InitializeProtocol() {
SetDeviceState(kDeviceStateIdle);
});
});
protocol_->OnIncomingJson([this, display](const cJSON* root) {
// Parse JSON data
auto type = cJSON_GetObjectItem(root, "type");
if (strcmp(type->valuestring, "tts") == 0) {
auto state = cJSON_GetObjectItem(root, "state");
if (strcmp(state->valuestring, "start") == 0) {
if (strcmp(state->valuestring, "thinking") == 0) {
Schedule([this]() { SetDeviceState(kDeviceStateThinking); });
} else if (strcmp(state->valuestring, "start") == 0) {
audio_service_.ResetDecoder();
accepting_tts_audio_.store(true);
Schedule([this]() {
aborted_ = false;
SetDeviceState(kDeviceStateSpeaking);
});
} else if (strcmp(state->valuestring, "stop") == 0) {
accepting_tts_audio_.store(false);
Schedule([this]() {
if (GetDeviceState() == kDeviceStateSpeaking) {
auto state = GetDeviceState();
if (state == kDeviceStateSpeaking || state == kDeviceStateThinking) {
if (listening_mode_ == kListeningModeManualStop) {
SetDeviceState(kDeviceStateIdle);
} else {
@ -597,9 +606,7 @@ void Application::InitializeProtocol() {
ESP_LOGI(TAG, "System command: %s", command->valuestring);
if (strcmp(command->valuestring, "reboot") == 0) {
// Do a reboot if user requests a OTA update
Schedule([this]() {
Reboot();
});
Schedule([this]() { Reboot(); });
} else {
ESP_LOGW(TAG, "Unknown system command: %s", command->valuestring);
}
@ -609,7 +616,8 @@ void Application::InitializeProtocol() {
auto message = cJSON_GetObjectItem(root, "message");
auto emotion = cJSON_GetObjectItem(root, "emotion");
if (cJSON_IsString(status) && cJSON_IsString(message) && cJSON_IsString(emotion)) {
Alert(status->valuestring, message->valuestring, emotion->valuestring, Lang::Sounds::OGG_VIBRATION);
Alert(status->valuestring, message->valuestring, emotion->valuestring,
Lang::Sounds::OGG_VIBRATION);
} else {
ESP_LOGW(TAG, "Alert command requires status, message and emotion");
}
@ -618,9 +626,10 @@ void Application::InitializeProtocol() {
auto payload = cJSON_GetObjectItem(root, "payload");
ESP_LOGI(TAG, "Received custom message: %s", cJSON_PrintUnformatted(root));
if (cJSON_IsObject(payload)) {
Schedule([this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() {
display->SetChatMessage("system", payload_str.c_str());
});
Schedule(
[this, display, payload_str = std::string(cJSON_PrintUnformatted(payload))]() {
display->SetChatMessage("system", payload_str.c_str());
});
} else {
ESP_LOGW(TAG, "Invalid custom message format: missing payload");
}
@ -629,7 +638,7 @@ void Application::InitializeProtocol() {
ESP_LOGW(TAG, "Unknown message type: %s", type->valuestring);
}
});
protocol_->Start();
}
@ -638,32 +647,27 @@ void Application::ShowActivationCode(const std::string& code, const std::string&
char digit;
const std::string_view& sound;
};
static const std::array<digit_sound, 10> digit_sounds{{
digit_sound{'0', Lang::Sounds::OGG_0},
digit_sound{'1', Lang::Sounds::OGG_1},
digit_sound{'2', Lang::Sounds::OGG_2},
digit_sound{'3', Lang::Sounds::OGG_3},
digit_sound{'4', Lang::Sounds::OGG_4},
digit_sound{'5', Lang::Sounds::OGG_5},
digit_sound{'6', Lang::Sounds::OGG_6},
digit_sound{'7', Lang::Sounds::OGG_7},
digit_sound{'8', Lang::Sounds::OGG_8},
digit_sound{'9', Lang::Sounds::OGG_9}
}};
static const std::array<digit_sound, 10> digit_sounds{
{digit_sound{'0', Lang::Sounds::OGG_0}, digit_sound{'1', Lang::Sounds::OGG_1},
digit_sound{'2', Lang::Sounds::OGG_2}, digit_sound{'3', Lang::Sounds::OGG_3},
digit_sound{'4', Lang::Sounds::OGG_4}, digit_sound{'5', Lang::Sounds::OGG_5},
digit_sound{'6', Lang::Sounds::OGG_6}, digit_sound{'7', Lang::Sounds::OGG_7},
digit_sound{'8', Lang::Sounds::OGG_8}, digit_sound{'9', Lang::Sounds::OGG_9}}};
// This sentence uses 9KB of SRAM, so we need to wait for it to finish
Alert(Lang::Strings::ACTIVATION, message.c_str(), "link", Lang::Sounds::OGG_ACTIVATION);
for (const auto& digit : code) {
auto it = std::find_if(digit_sounds.begin(), digit_sounds.end(),
[digit](const digit_sound& ds) { return ds.digit == digit; });
[digit](const digit_sound& ds) { return ds.digit == digit; });
if (it != digit_sounds.end()) {
audio_service_.PlaySound(it->sound);
}
}
}
void Application::Alert(const char* status, const char* message, const char* emotion, const std::string_view& sound) {
void Application::Alert(const char* status, const char* message, const char* emotion,
const std::string_view& sound) {
ESP_LOGW(TAG, "Alert [%s] %s: %s", emotion, status, message);
auto display = Board::GetInstance().GetDisplay();
display->SetStatus(status);
@ -683,9 +687,7 @@ void Application::DismissAlert() {
}
}
void Application::ToggleChatState() {
ToggleChatStateForMode(kChatAgentModeNormal, false);
}
void Application::ToggleChatState() { ToggleChatStateForMode(kChatAgentModeNormal, false); }
void Application::ToggleChatStateWithVision() {
ToggleChatStateForMode(kChatAgentModeNormal, true);
@ -698,9 +700,7 @@ void Application::ToggleChatStateForMode(ChatAgentMode agent_mode, bool vision_e
xEventGroupSetBits(event_group_, MAIN_EVENT_TOGGLE_CHAT);
}
bool Application::IsVisionTextModeEnabled() const {
return vision_text_mode_enabled_.load();
}
bool Application::IsVisionTextModeEnabled() const { return vision_text_mode_enabled_.load(); }
const char* Application::GetChatAgentModeName() const {
return chat_agent_mode_.load() == kChatAgentModeBeaver ? "beaver" : "normal";
@ -720,9 +720,7 @@ void Application::StartListening() {
xEventGroupSetBits(event_group_, MAIN_EVENT_START_LISTENING);
}
void Application::StopListening() {
xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING);
}
void Application::StopListening() { xEventGroupSetBits(event_group_, MAIN_EVENT_STOP_LISTENING); }
void Application::HandleToggleChatEvent() {
auto state = GetDeviceState();
@ -748,21 +746,21 @@ void Application::HandleToggleChatEvent() {
if (state == kDeviceStateIdle) {
ListeningMode mode = GetDefaultListeningMode();
bool agent_mode_changed = chat_agent_mode_.load() != active_chat_agent_mode_.load();
bool vision_mode_changed = vision_text_mode_enabled_.load() != active_vision_text_mode_enabled_.load();
bool vision_mode_changed =
vision_text_mode_enabled_.load() != active_vision_text_mode_enabled_.load();
if (!protocol_->IsAudioChannelOpened() || agent_mode_changed || vision_mode_changed) {
if (protocol_->IsAudioChannelOpened()) {
protocol_->CloseAudioChannel();
}
SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update)
Schedule([this, mode]() {
ContinueOpenAudioChannel(mode);
});
Schedule([this, mode]() { ContinueOpenAudioChannel(mode); });
return;
}
SetListeningMode(mode);
} else if (state == kDeviceStateSpeaking) {
} else if (state == kDeviceStateSpeaking || state == kDeviceStateThinking) {
AbortSpeaking(kAbortReasonNone);
SetListeningMode(GetDefaultListeningMode());
} else if (state == kDeviceStateListening) {
protocol_->CloseAudioChannel();
}
@ -791,7 +789,7 @@ void Application::ContinueOpenAudioChannel(ListeningMode mode) {
void Application::HandleStartListeningEvent() {
auto state = GetDeviceState();
if (state == kDeviceStateActivating) {
SetDeviceState(kDeviceStateIdle);
return;
@ -805,18 +803,16 @@ void Application::HandleStartListeningEvent() {
ESP_LOGE(TAG, "Protocol not initialized");
return;
}
if (state == kDeviceStateIdle) {
if (!protocol_->IsAudioChannelOpened()) {
SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update)
Schedule([this]() {
ContinueOpenAudioChannel(kListeningModeManualStop);
});
Schedule([this]() { ContinueOpenAudioChannel(kListeningModeManualStop); });
return;
}
SetListeningMode(kListeningModeManualStop);
} else if (state == kDeviceStateSpeaking) {
} else if (state == kDeviceStateSpeaking || state == kDeviceStateThinking) {
AbortSpeaking(kAbortReasonNone);
SetListeningMode(kListeningModeManualStop);
}
@ -824,7 +820,7 @@ void Application::HandleStartListeningEvent() {
void Application::HandleStopListeningEvent() {
auto state = GetDeviceState();
if (state == kDeviceStateAudioTesting) {
audio_service_.EnableAudioTesting(false);
SetDeviceState(kDeviceStateWifiConfiguring);
@ -854,17 +850,14 @@ void Application::HandleWakeWordDetectedEvent() {
SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update),
// then continue with OpenAudioChannel which may block for ~1 second
Schedule([this, wake_word]() {
ContinueWakeWordInvoke(wake_word);
});
Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); });
return;
}
// Channel already opened, continue directly
ContinueWakeWordInvoke(wake_word);
} else if (state == kDeviceStateSpeaking || state == kDeviceStateListening) {
} else if (state == kDeviceStateSpeaking || state == kDeviceStateThinking ||
state == kDeviceStateListening) {
AbortSpeaking(kAbortReasonWakeWordDetected);
// Clear send queue to avoid sending residues to server
while (audio_service_.PopPacketFromSendQueue());
if (state == kDeviceStateListening) {
protocol_->SendStartListening(GetDefaultListeningMode());
@ -925,14 +918,14 @@ void Application::HandleStateChangedEvent() {
auto display = board.GetDisplay();
auto led = board.GetLed();
led->OnStateChanged();
switch (new_state) {
case kDeviceStateUnknown:
case kDeviceStateIdle:
vision_frame_sent_for_current_listen_.store(false);
display->SetStatus(Lang::Strings::STANDBY);
display->ClearChatMessages(); // Clear messages first
display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count)
display->ClearChatMessages(); // Clear messages first
display->SetEmotion("neutral"); // Then set emotion (wechat mode checks child count)
audio_service_.EnableVoiceProcessing(false);
audio_service_.EnableWakeWordDetection(true);
break;
@ -947,18 +940,14 @@ void Application::HandleStateChangedEvent() {
display->SetStatus(Lang::Strings::LISTENING);
display->SetEmotion("neutral");
// Make sure the audio processor is running
if (play_popup_on_listening_ || !audio_service_.IsAudioProcessorRunning()) {
// For auto mode, wait for playback queue to be empty before enabling voice processing
// This prevents audio truncation when STOP arrives late due to network jitter
if (listening_mode_ == kListeningModeAutoStop) {
audio_service_.WaitForPlaybackQueueEmpty();
}
// Send the start listening command
protocol_->SendStartListening(listening_mode_);
audio_service_.EnableVoiceProcessing(true);
// Re-entering listening after an interrupt must restart the capture path even if the
// processor task is still marked running, otherwise realtime mode can show Listening
// while no fresh mic frames are sent.
if (listening_mode_ == kListeningModeAutoStop) {
audio_service_.WaitForPlaybackQueueEmpty();
}
protocol_->SendStartListening(listening_mode_);
audio_service_.EnableVoiceProcessing(true);
#ifdef CONFIG_WAKE_WORD_DETECTION_IN_LISTENING
// Enable wake word detection in listening mode (configured via Kconfig)
@ -967,13 +956,23 @@ void Application::HandleStateChangedEvent() {
// Disable wake word detection in listening mode
audio_service_.EnableWakeWordDetection(false);
#endif
// Play popup sound after ResetDecoder (in EnableVoiceProcessing) has been called
if (play_popup_on_listening_) {
play_popup_on_listening_ = false;
audio_service_.PlaySound(Lang::Sounds::OGG_POPUP);
}
break;
case kDeviceStateThinking:
vad_speaking_.store(false);
display->SetStatus(Lang::Strings::THINKING);
display->SetEmotion("thinking");
if (listening_mode_ != kListeningModeRealtime) {
audio_service_.EnableVoiceProcessing(false);
audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
}
break;
case kDeviceStateSpeaking:
display->SetStatus(Lang::Strings::SPEAKING);
@ -982,7 +981,9 @@ void Application::HandleStateChangedEvent() {
// Only AFE wake word can be detected in speaking mode
audio_service_.EnableWakeWordDetection(audio_service_.IsAfeWakeWord());
}
audio_service_.ResetDecoder();
if (!accepting_tts_audio_.load()) {
audio_service_.ResetDecoder();
}
break;
case kDeviceStateWifiConfiguring:
audio_service_.EnableVoiceProcessing(false);
@ -1026,6 +1027,8 @@ void Application::Schedule(std::function<void()>&& callback) {
void Application::AbortSpeaking(AbortReason reason) {
ESP_LOGI(TAG, "Abort speaking");
aborted_ = true;
accepting_tts_audio_.store(false);
audio_service_.ResetDecoder();
if (protocol_) {
protocol_->SendAbortSpeaking(reason);
}
@ -1069,7 +1072,8 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver
}
ESP_LOGI(TAG, "Starting firmware upgrade from URL: %s", upgrade_url.c_str());
Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download", Lang::Sounds::OGG_UPGRADE);
Alert(Lang::Strings::OTA_UPGRADE, Lang::Strings::UPGRADING, "download",
Lang::Sounds::OGG_UPGRADE);
vTaskDelay(pdMS_TO_TICKS(3000));
SetDeviceState(kDeviceStateUpgrading);
@ -1091,17 +1095,19 @@ bool Application::UpgradeFirmware(const std::string& url, const std::string& ver
if (!upgrade_success) {
// Upgrade failed, restart audio service and continue running
ESP_LOGE(TAG, "Firmware upgrade failed, restarting audio service and continuing operation...");
audio_service_.Start(); // Restart audio service
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level
Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark", Lang::Sounds::OGG_EXCLAMATION);
ESP_LOGE(TAG,
"Firmware upgrade failed, restarting audio service and continuing operation...");
audio_service_.Start(); // Restart audio service
board.SetPowerSaveLevel(PowerSaveLevel::LOW_POWER); // Restore power save level
Alert(Lang::Strings::ERROR, Lang::Strings::UPGRADE_FAILED, "circle_xmark",
Lang::Sounds::OGG_EXCLAMATION);
vTaskDelay(pdMS_TO_TICKS(3000));
return false;
} else {
// Upgrade success, reboot immediately
ESP_LOGI(TAG, "Firmware upgrade successful, rebooting...");
display->SetChatMessage("system", "Upgrade successful, rebooting...");
vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message
vTaskDelay(pdMS_TO_TICKS(1000)); // Brief pause to show message
Reboot();
return true;
}
@ -1113,25 +1119,21 @@ void Application::WakeWordInvoke(const std::string& wake_word) {
}
auto state = GetDeviceState();
if (state == kDeviceStateIdle) {
audio_service_.EncodeWakeWord();
if (!protocol_->IsAudioChannelOpened()) {
SetDeviceState(kDeviceStateConnecting);
// Schedule to let the state change be processed first (UI update)
Schedule([this, wake_word]() {
ContinueWakeWordInvoke(wake_word);
});
Schedule([this, wake_word]() { ContinueWakeWordInvoke(wake_word); });
return;
}
// Channel already opened, continue directly
ContinueWakeWordInvoke(wake_word);
} else if (state == kDeviceStateSpeaking) {
Schedule([this]() {
AbortSpeaking(kAbortReasonNone);
});
} else if (state == kDeviceStateListening) {
} else if (state == kDeviceStateSpeaking || state == kDeviceStateThinking) {
Schedule([this]() { AbortSpeaking(kAbortReasonNone); });
} else if (state == kDeviceStateListening) {
Schedule([this]() {
if (protocol_) {
protocol_->CloseAudioChannel();
@ -1163,7 +1165,7 @@ void Application::RegisterMcpBroadcastCallback(std::function<void(const std::str
void Application::SendMcpMessage(const std::string& payload) {
// Always schedule to run in main task for thread safety
Schedule([this, payload](){
Schedule([this, payload]() {
if (protocol_) {
protocol_->SendMcpMessage(payload);
}
@ -1179,18 +1181,18 @@ void Application::SetAecMode(AecMode mode) {
auto& board = Board::GetInstance();
auto display = board.GetDisplay();
switch (aec_mode_) {
case kAecOff:
audio_service_.EnableDeviceAec(false);
display->ShowNotification(Lang::Strings::RTC_MODE_OFF);
break;
case kAecOnServerSide:
audio_service_.EnableDeviceAec(false);
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
break;
case kAecOnDeviceSide:
audio_service_.EnableDeviceAec(true);
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
break;
case kAecOff:
audio_service_.EnableDeviceAec(false);
display->ShowNotification(Lang::Strings::RTC_MODE_OFF);
break;
case kAecOnServerSide:
audio_service_.EnableDeviceAec(false);
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
break;
case kAecOnDeviceSide:
audio_service_.EnableDeviceAec(true);
display->ShowNotification(Lang::Strings::RTC_MODE_ON);
break;
}
// If the AEC mode is changed, close the audio channel
@ -1200,9 +1202,7 @@ void Application::SetAecMode(AecMode mode) {
});
}
void Application::PlaySound(const std::string_view& sound) {
audio_service_.PlaySound(sound);
}
void Application::PlaySound(const std::string_view& sound) { audio_service_.PlaySound(sound); }
void Application::ResetProtocol() {
Schedule([this]() {