Compare commits
2 Commits
2c4329fd84
...
66d318774a
| Author | SHA1 | Date | |
|---|---|---|---|
| 66d318774a | |||
| 60df0fe196 |
@ -1,6 +1,5 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@ -12,7 +11,7 @@ import traceback
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from typing import Any, Coroutine, Optional
|
||||
|
||||
import httpx
|
||||
import opuslib
|
||||
@ -67,6 +66,7 @@ TTS_START_CONSECUTIVE_AUDIBLE_FRAMES = int(os.getenv("TTS_START_CONSECUTIVE_AUDI
|
||||
TTS_INTERRUPT_SILENCE_FRAMES = 3
|
||||
INTERRUPT_TOPIC = "lk.interrupt"
|
||||
VISION_FRAME_TOPIC = "vision.frame"
|
||||
MCP_TOPIC = "mcp"
|
||||
AGENT_STATE_ATTRIBUTE = "lk.agent.state"
|
||||
TTS_DISPLAY_SENTENCE_BREAKS = "。!?!?;;"
|
||||
TTS_DISPLAY_SCROLL_WIDTH = int(os.getenv("TTS_DISPLAY_SCROLL_WIDTH", "18"))
|
||||
@ -122,6 +122,9 @@ class DeviceSession:
|
||||
last_interrupt_time: float = 0.0
|
||||
last_uplink_audible_time: float = 0.0
|
||||
agent_dispatch_task: Optional[asyncio.Task] = None
|
||||
room_connect_task: Optional[asyncio.Task] = None
|
||||
background_tasks: set[asyncio.Task[Any]] = field(default_factory=set)
|
||||
forwarding_track_participants: dict[str, str] = field(default_factory=dict)
|
||||
closed: bool = False
|
||||
captured_frame_count: int = 0
|
||||
first_capture_log_time: float = 0.0
|
||||
@ -208,11 +211,15 @@ class ESP32LiveKitBridge:
|
||||
session: DeviceSession,
|
||||
task: asyncio.Task[Any],
|
||||
) -> None:
|
||||
if session.room_connect_task is task:
|
||||
session.room_connect_task = None
|
||||
if task.cancelled():
|
||||
return
|
||||
try:
|
||||
task.result()
|
||||
except Exception as exc:
|
||||
if session.closed:
|
||||
return
|
||||
self._log_exception(
|
||||
f"LiveKit 房间连接后台任务失败: room={session.room_name}",
|
||||
exc,
|
||||
@ -221,6 +228,48 @@ class ESP32LiveKitBridge:
|
||||
if websocket is not None:
|
||||
asyncio.create_task(websocket.close(code=1011, reason="livekit connect failed"))
|
||||
|
||||
def _create_session_task(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
coroutine: Coroutine[Any, Any, Any],
|
||||
description: str,
|
||||
) -> Optional[asyncio.Task[Any]]:
|
||||
if session.closed:
|
||||
coroutine.close()
|
||||
return None
|
||||
|
||||
task = asyncio.create_task(coroutine)
|
||||
session.background_tasks.add(task)
|
||||
task.add_done_callback(
|
||||
lambda done_task: self._handle_session_task_done(
|
||||
session,
|
||||
done_task,
|
||||
description,
|
||||
)
|
||||
)
|
||||
return task
|
||||
|
||||
def _handle_session_task_done(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
task: asyncio.Task[Any],
|
||||
description: str,
|
||||
) -> None:
|
||||
session.background_tasks.discard(task)
|
||||
if task.cancelled():
|
||||
return
|
||||
try:
|
||||
task.result()
|
||||
except Exception as exc:
|
||||
if not session.closed:
|
||||
self._log_exception(f"{description} 失败: room={session.room_name}", exc)
|
||||
|
||||
async def _disconnect_room_quietly(self, session: DeviceSession, reason: str) -> None:
|
||||
try:
|
||||
await session.room.disconnect()
|
||||
except Exception as exc:
|
||||
print(f"断开 LiveKit 房间失败: room={session.room_name} reason={reason} error={exc}")
|
||||
|
||||
async def _capture_mic_frame(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
@ -522,6 +571,8 @@ class ESP32LiveKitBridge:
|
||||
return False
|
||||
|
||||
async def _send_agent_interrupt(self, session: DeviceSession, reason: str) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
payload = {
|
||||
"type": "interrupt",
|
||||
"topic": INTERRUPT_TOPIC,
|
||||
@ -552,6 +603,8 @@ class ESP32LiveKitBridge:
|
||||
return path
|
||||
|
||||
async def _publish_vision_frame(self, session: DeviceSession, message: dict[str, Any]) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
image = message.get("image")
|
||||
if not isinstance(image, str) or not image:
|
||||
print("收到 vision frame,但 image 字段为空")
|
||||
@ -601,7 +654,81 @@ class ESP32LiveKitBridge:
|
||||
if last_error is not None:
|
||||
print(f"发送 vision frame 失败,publish_data 签名不兼容: {last_error}")
|
||||
|
||||
async def _publish_mcp_message(self, session: DeviceSession, message: dict[str, Any]) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
payload = message.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
print(f"收到 ESP32 MCP 消息但缺少 payload: {message}")
|
||||
return
|
||||
|
||||
participant = getattr(session.room, "local_participant", None)
|
||||
if participant is None:
|
||||
print("跳过发送 MCP 消息,local participant 尚未就绪")
|
||||
return
|
||||
|
||||
outbound = {
|
||||
"type": "mcp",
|
||||
"topic": MCP_TOPIC,
|
||||
"room": session.room_name,
|
||||
"identity": session.identity,
|
||||
"device_id": session.device_id,
|
||||
"payload": payload,
|
||||
}
|
||||
data = json.dumps(outbound, ensure_ascii=False).encode("utf-8")
|
||||
agent_identities = self._get_agent_identities(session)
|
||||
kwargs: dict[str, Any] = {}
|
||||
if agent_identities:
|
||||
kwargs["destination_identities"] = agent_identities
|
||||
|
||||
last_error: Optional[Exception] = None
|
||||
for attempt in ({"topic": MCP_TOPIC, **kwargs}, kwargs):
|
||||
try:
|
||||
await participant.publish_data(data, **attempt)
|
||||
print(
|
||||
f"已发送 MCP 响应: id={payload.get('id')} "
|
||||
f"targets={agent_identities or 'broadcast'}"
|
||||
)
|
||||
return
|
||||
except TypeError as exc:
|
||||
last_error = exc
|
||||
except Exception as exc:
|
||||
print(f"发送 MCP 响应失败: {exc}")
|
||||
return
|
||||
|
||||
if last_error is not None:
|
||||
print(f"发送 MCP 响应失败,publish_data 签名不兼容: {last_error}")
|
||||
|
||||
async def _forward_mcp_to_device(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
source_identity: str,
|
||||
) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
if session.websocket is None:
|
||||
print("跳过 MCP 请求,ESP32 尚未连接")
|
||||
return
|
||||
|
||||
await session.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "mcp",
|
||||
"payload": payload,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
print(
|
||||
f"已转发 MCP 请求到 ESP32: id={payload.get('id')} "
|
||||
f"method={payload.get('method')} source={source_identity}"
|
||||
)
|
||||
|
||||
async def _send_tts_state(self, session: DeviceSession, state: str) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
if session.websocket is None:
|
||||
print(f"跳过 tts {state},ESP32 尚未连接")
|
||||
return
|
||||
@ -609,6 +736,8 @@ class ESP32LiveKitBridge:
|
||||
print(f"已发送 tts {state}: device={session.device_id}")
|
||||
|
||||
async def _send_emotion(self, session: DeviceSession, emotion: str) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
if session.websocket is None:
|
||||
print(f"跳过 emotion {emotion},ESP32 尚未连接")
|
||||
return
|
||||
@ -634,6 +763,8 @@ class ESP32LiveKitBridge:
|
||||
await self._send_emotion(session, emotion)
|
||||
|
||||
async def _send_tts_text(self, session: DeviceSession, text: str, final: bool) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
if session.websocket is None:
|
||||
return
|
||||
raw_text = text
|
||||
@ -664,12 +795,18 @@ class ESP32LiveKitBridge:
|
||||
|
||||
if len(text) <= TTS_DISPLAY_SCROLL_WIDTH:
|
||||
self._cancel_tts_display_task(session)
|
||||
asyncio.create_task(self._send_tts_text(session, text, final))
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._send_tts_text(session, text, final),
|
||||
"发送 TTS 字幕",
|
||||
)
|
||||
return
|
||||
|
||||
if session.tts_display_task is None or session.tts_display_task.done():
|
||||
session.tts_display_task = asyncio.create_task(
|
||||
self._scroll_tts_display_text(session, session.tts_stream_id)
|
||||
session.tts_display_task = self._create_session_task(
|
||||
session,
|
||||
self._scroll_tts_display_text(session, session.tts_stream_id),
|
||||
"滚动 TTS 字幕",
|
||||
)
|
||||
|
||||
async def _scroll_tts_display_text(self, session: DeviceSession, stream_id: int) -> None:
|
||||
@ -788,7 +925,11 @@ class ESP32LiveKitBridge:
|
||||
f"[agent-state] room={session.room_name} identity={participant.identity} state={state}"
|
||||
)
|
||||
if state == "thinking":
|
||||
asyncio.create_task(self._start_thinking(session))
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._start_thinking(session),
|
||||
"处理 agent thinking 状态",
|
||||
)
|
||||
|
||||
async def _stop_tts(self, session: DeviceSession) -> None:
|
||||
if not session.tts_active and not session.tts_thinking:
|
||||
@ -829,14 +970,20 @@ class ESP32LiveKitBridge:
|
||||
session.tts_suppressed_until = now + TTS_INTERRUPT_SUPPRESS_SECONDS
|
||||
session.tts_waiting_for_user_audio_after_interrupt = True
|
||||
await self._force_stop_tts(session, reason)
|
||||
asyncio.create_task(self._send_agent_interrupt(session, reason))
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._send_agent_interrupt(session, reason),
|
||||
"发送 agent interrupt",
|
||||
)
|
||||
|
||||
def _reset_tts_idle_timer(self, session: DeviceSession) -> None:
|
||||
session.tts_last_audible_at = time.monotonic()
|
||||
if session.tts_idle_task is not None:
|
||||
session.tts_idle_task.cancel()
|
||||
session.tts_idle_task = asyncio.create_task(
|
||||
self._tts_idle_watchdog(session, session.tts_stream_id)
|
||||
session.tts_idle_task = self._create_session_task(
|
||||
session,
|
||||
self._tts_idle_watchdog(session, session.tts_stream_id),
|
||||
"TTS idle watchdog",
|
||||
)
|
||||
|
||||
async def _tts_idle_watchdog(self, session: DeviceSession, stream_id: int) -> None:
|
||||
@ -890,6 +1037,8 @@ class ESP32LiveKitBridge:
|
||||
participant: rtc.RemoteParticipant,
|
||||
source: str,
|
||||
) -> None:
|
||||
if session.closed or session.websocket is None:
|
||||
return
|
||||
if track.kind != rtc.TrackKind.KIND_AUDIO:
|
||||
return
|
||||
|
||||
@ -905,6 +1054,7 @@ class ESP32LiveKitBridge:
|
||||
if existing_task is not None and existing_task.done():
|
||||
print(f"检测到已结束的音频转发任务,重新创建: sid={track_sid}")
|
||||
session.forwarding_tracks.pop(track_sid, None)
|
||||
session.forwarding_track_participants.pop(track_sid, None)
|
||||
|
||||
task = asyncio.create_task(
|
||||
self.forward_audio_to_esp32(
|
||||
@ -914,17 +1064,37 @@ class ESP32LiveKitBridge:
|
||||
)
|
||||
)
|
||||
session.forwarding_tracks[track_sid] = task
|
||||
session.forwarding_track_participants[track_sid] = participant.identity
|
||||
print(
|
||||
f"收到音频流: {participant.identity} sid={track_sid} "
|
||||
f"source={source} room={session.room_name}"
|
||||
)
|
||||
|
||||
def _cancel_forwarding_tracks(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
participant_identity: Optional[str] = None,
|
||||
) -> list[asyncio.Task[Any]]:
|
||||
cancelled: list[asyncio.Task[Any]] = []
|
||||
for track_sid, task in list(session.forwarding_tracks.items()):
|
||||
track_participant = session.forwarding_track_participants.get(track_sid)
|
||||
if participant_identity is not None and track_participant != participant_identity:
|
||||
continue
|
||||
session.forwarding_tracks.pop(track_sid, None)
|
||||
session.forwarding_track_participants.pop(track_sid, None)
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
cancelled.append(task)
|
||||
return cancelled
|
||||
|
||||
def _scan_participant_audio_tracks(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
participant: rtc.RemoteParticipant,
|
||||
source: str,
|
||||
) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
publications = getattr(participant, "track_publications", None) or {}
|
||||
for publication in publications.values():
|
||||
track = getattr(publication, "track", None)
|
||||
@ -992,6 +1162,76 @@ class ESP32LiveKitBridge:
|
||||
|
||||
return normalized[start:].strip() or normalized
|
||||
|
||||
async def _handle_room_connected(self, session: DeviceSession) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
print(f"✅ 成功连接到 LiveKit 房间: room={session.room_name}")
|
||||
self._log_agent_participants(session, "connected")
|
||||
for participant in list(session.room.remote_participants.values()):
|
||||
if session.closed:
|
||||
return
|
||||
if self._is_agent_participant(participant, session.agent_name):
|
||||
session.agent_ready.set()
|
||||
self._scan_participant_audio_tracks(session, participant, "connected_scan")
|
||||
self._handle_agent_state(session, participant)
|
||||
|
||||
async def _handle_participant_connected(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
participant: rtc.RemoteParticipant,
|
||||
) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
role = "Agent" if self._is_agent_participant(participant, session.agent_name) else "Remote participant"
|
||||
print(f"👋 {role} ({participant.identity}) 已加入房间: room={session.room_name}")
|
||||
self._log_agent_participants(session, "participant_connected")
|
||||
if self._is_agent_participant(participant, session.agent_name):
|
||||
session.agent_ready.set()
|
||||
self._scan_participant_audio_tracks(
|
||||
session, participant, "participant_connected_scan"
|
||||
)
|
||||
self._handle_agent_state(session, participant)
|
||||
|
||||
async def _handle_participant_attributes_changed(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
changed: list[str],
|
||||
participant: rtc.Participant,
|
||||
) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
if AGENT_STATE_ATTRIBUTE not in changed:
|
||||
return
|
||||
if not isinstance(participant, rtc.RemoteParticipant):
|
||||
return
|
||||
if not self._is_agent_participant(participant, session.agent_name):
|
||||
return
|
||||
self._handle_agent_state(session, participant)
|
||||
|
||||
async def _handle_track_subscribed(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
track: rtc.Track,
|
||||
publication: rtc.TrackPublication,
|
||||
participant: rtc.RemoteParticipant,
|
||||
source: str,
|
||||
) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
self._maybe_forward_remote_audio(session, track, publication, participant, source)
|
||||
|
||||
async def _handle_track_published(
|
||||
self,
|
||||
session: DeviceSession,
|
||||
publication: rtc.RemoteTrackPublication,
|
||||
participant: rtc.RemoteParticipant,
|
||||
) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
track = getattr(publication, "track", None)
|
||||
if track is not None:
|
||||
self._maybe_forward_remote_audio(session, track, publication, participant, "published")
|
||||
|
||||
def _register_room_handlers(self, session: DeviceSession) -> None:
|
||||
@session.room.on("connection_state_changed")
|
||||
def on_connection_state_changed(state: int) -> None:
|
||||
@ -1000,55 +1240,69 @@ class ESP32LiveKitBridge:
|
||||
|
||||
@session.room.on("connected")
|
||||
def on_connected() -> None:
|
||||
print(f"✅ 成功连接到 LiveKit 房间: room={session.room_name}")
|
||||
self._log_agent_participants(session, "connected")
|
||||
for participant in session.room.remote_participants.values():
|
||||
if self._is_agent_participant(participant, session.agent_name):
|
||||
session.agent_ready.set()
|
||||
self._scan_participant_audio_tracks(session, participant, "connected_scan")
|
||||
self._handle_agent_state(session, participant)
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._handle_room_connected(session),
|
||||
"处理 LiveKit connected 事件",
|
||||
)
|
||||
|
||||
@session.room.on("participant_connected")
|
||||
def on_participant_connected(participant: rtc.RemoteParticipant) -> None:
|
||||
role = "Agent" if self._is_agent_participant(participant, session.agent_name) else "Remote participant"
|
||||
print(f"👋 {role} ({participant.identity}) 已加入房间: room={session.room_name}")
|
||||
self._log_agent_participants(session, "participant_connected")
|
||||
if self._is_agent_participant(participant, session.agent_name):
|
||||
session.agent_ready.set()
|
||||
self._scan_participant_audio_tracks(
|
||||
session, participant, "participant_connected_scan"
|
||||
)
|
||||
self._handle_agent_state(session, participant)
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._handle_participant_connected(session, participant),
|
||||
"处理 LiveKit participant_connected 事件",
|
||||
)
|
||||
|
||||
@session.room.on("participant_disconnected")
|
||||
def on_participant_disconnected(participant: rtc.RemoteParticipant) -> None:
|
||||
print(f"👋 远端参与者离开房间: room={session.room_name} identity={participant.identity}")
|
||||
session.forwarding_tracks = {
|
||||
track_sid: task
|
||||
for track_sid, task in session.forwarding_tracks.items()
|
||||
if not track_sid.endswith(f":{participant.identity}")
|
||||
}
|
||||
self._cancel_forwarding_tracks(session, participant.identity)
|
||||
|
||||
@session.room.on("participant_attributes_changed")
|
||||
def on_participant_attributes_changed(changed: list[str], participant: rtc.Participant) -> None:
|
||||
if AGENT_STATE_ATTRIBUTE not in changed:
|
||||
return
|
||||
if not isinstance(participant, rtc.RemoteParticipant):
|
||||
return
|
||||
if not self._is_agent_participant(participant, session.agent_name):
|
||||
return
|
||||
self._handle_agent_state(session, participant)
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._handle_participant_attributes_changed(session, changed, participant),
|
||||
"处理 LiveKit participant_attributes_changed 事件",
|
||||
)
|
||||
|
||||
@session.room.on("data_received")
|
||||
def on_data_received(data_packet: rtc.DataPacket) -> None:
|
||||
identity = data_packet.participant.identity if data_packet.participant else "未知"
|
||||
packet_topic = getattr(data_packet, "topic", None)
|
||||
try:
|
||||
decoded = data_packet.data.decode("utf-8")
|
||||
print(
|
||||
f"📩 [数据接收 | room={session.room_name} | {identity}]: "
|
||||
f"{data_packet.data.decode('utf-8')}"
|
||||
f"{decoded}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
decoded = ""
|
||||
|
||||
try:
|
||||
payload = json.loads(decoded) if decoded else None
|
||||
except Exception:
|
||||
payload = None
|
||||
|
||||
if isinstance(payload, dict) and (
|
||||
packet_topic == MCP_TOPIC
|
||||
or payload.get("type") == "mcp"
|
||||
or payload.get("topic") == MCP_TOPIC
|
||||
):
|
||||
mcp_payload = payload.get("payload")
|
||||
if isinstance(mcp_payload, dict):
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._forward_mcp_to_device(
|
||||
session,
|
||||
mcp_payload,
|
||||
source_identity=identity,
|
||||
),
|
||||
"转发 MCP 到 ESP32",
|
||||
)
|
||||
else:
|
||||
print(f"收到 MCP 数据但缺少 payload: {payload}")
|
||||
|
||||
@session.room.on("transcription_received")
|
||||
def on_transcription_received(
|
||||
@ -1074,7 +1328,11 @@ class ESP32LiveKitBridge:
|
||||
)
|
||||
if emotion and emotion != session.tts_emotion:
|
||||
session.tts_emotion = emotion
|
||||
asyncio.create_task(self._send_emotion(session, emotion))
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._send_emotion(session, emotion),
|
||||
"发送 emotion",
|
||||
)
|
||||
display_text = self._current_tts_display_text(tts_text)
|
||||
print(f"[livekit-llm] display_text={display_text!r} final={segment.final}")
|
||||
if not display_text or display_text == session.tts_transcript_text:
|
||||
@ -1084,14 +1342,19 @@ class ESP32LiveKitBridge:
|
||||
if not segment.final:
|
||||
continue
|
||||
display_text = segment.text
|
||||
asyncio.create_task(self._start_thinking(session))
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._start_thinking(session),
|
||||
"发送 TTS thinking",
|
||||
)
|
||||
|
||||
if session.websocket is not None:
|
||||
ws = session.websocket
|
||||
if is_agent:
|
||||
self._update_tts_display_text(session, display_text, segment.final)
|
||||
else:
|
||||
asyncio.create_task(
|
||||
self._create_session_task(
|
||||
session,
|
||||
ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
@ -1100,7 +1363,8 @@ class ESP32LiveKitBridge:
|
||||
"final": segment.final,
|
||||
}
|
||||
)
|
||||
)
|
||||
),
|
||||
"发送 STT 到 ESP32",
|
||||
)
|
||||
|
||||
@session.room.on("track_subscribed")
|
||||
@ -1109,7 +1373,11 @@ class ESP32LiveKitBridge:
|
||||
publication: rtc.TrackPublication,
|
||||
participant: rtc.RemoteParticipant,
|
||||
) -> None:
|
||||
self._maybe_forward_remote_audio(session, track, publication, participant, "event")
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._handle_track_subscribed(session, track, publication, participant, "event"),
|
||||
"处理 LiveKit track_subscribed 事件",
|
||||
)
|
||||
|
||||
@session.room.on("track_published")
|
||||
def on_track_published(
|
||||
@ -1121,18 +1389,25 @@ class ESP32LiveKitBridge:
|
||||
# f"📡 远端音轨已发布: room={session.room_name} identity={participant.identity} "
|
||||
# f"track_sid={track_sid}"
|
||||
# )
|
||||
track = getattr(publication, "track", None)
|
||||
if track is not None:
|
||||
self._maybe_forward_remote_audio(session, track, publication, participant, "published")
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._handle_track_published(session, publication, participant),
|
||||
"处理 LiveKit track_published 事件",
|
||||
)
|
||||
|
||||
async def _connect_session_room(self, session: DeviceSession) -> None:
|
||||
if session.closed:
|
||||
return
|
||||
self._register_room_handlers(session)
|
||||
connected = False
|
||||
|
||||
# print(f"[config] livekit_ws_url={LIVEKIT_WS_URL}")
|
||||
# print(f"[config] token_url={TOKEN_URL}")
|
||||
# print(f"[config] room={session.room_name} identity={session.identity}")
|
||||
# print(f"[config] livekit_connect_timeout={CONNECT_TIMEOUT_SECONDS}")
|
||||
token = await fetch_token(session.room_name, session.identity, session.agent_name)
|
||||
if session.closed:
|
||||
return
|
||||
|
||||
try:
|
||||
await session.room.connect(
|
||||
@ -1140,7 +1415,10 @@ class ESP32LiveKitBridge:
|
||||
token,
|
||||
options=rtc.RoomOptions(connect_timeout=CONNECT_TIMEOUT_SECONDS),
|
||||
)
|
||||
connected = True
|
||||
except Exception as exc:
|
||||
if session.closed:
|
||||
return
|
||||
self._log_exception(
|
||||
f"连接 LiveKit 房间失败: room={session.room_name}",
|
||||
exc,
|
||||
@ -1152,6 +1430,9 @@ class ESP32LiveKitBridge:
|
||||
)
|
||||
raise
|
||||
|
||||
if session.closed:
|
||||
await self._disconnect_room_quietly(session, "session_closed_after_connect")
|
||||
return
|
||||
print(f"已连接到 LiveKit 房间: {session.room.name}")
|
||||
# print(f"[livekit] local_identity={session.room.local_participant.identity}")
|
||||
# print(f"[livekit] local_sid={session.room.local_participant.sid}")
|
||||
@ -1159,6 +1440,9 @@ class ESP32LiveKitBridge:
|
||||
self._log_agent_participants(session, "after_connect")
|
||||
|
||||
await self.ensure_agent_dispatched(session)
|
||||
if session.closed:
|
||||
await self._disconnect_room_quietly(session, "session_closed_after_dispatch")
|
||||
return
|
||||
|
||||
track = rtc.LocalAudioTrack.create_audio_track(
|
||||
f"esp32-mic-{session.device_id}",
|
||||
@ -1166,6 +1450,9 @@ class ESP32LiveKitBridge:
|
||||
)
|
||||
options = rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_MICROPHONE)
|
||||
publication = await session.room.local_participant.publish_track(track, options)
|
||||
if session.closed:
|
||||
await self._disconnect_room_quietly(session, "session_closed_after_publish")
|
||||
return
|
||||
publication_sid = getattr(publication, "sid", None)
|
||||
track_sid = getattr(track, "sid", None)
|
||||
# print(
|
||||
@ -1185,6 +1472,9 @@ class ESP32LiveKitBridge:
|
||||
return
|
||||
print(f"⚠️ agent 等待超时: room={session.room_name}")
|
||||
|
||||
if connected and session.closed:
|
||||
await self._disconnect_room_quietly(session, "session_closed_after_agent_wait")
|
||||
|
||||
async def start(self) -> None:
|
||||
print(f"[config] websocket_port={WS_PORT}")
|
||||
print(f"[config] websocket_max_queue={WS_MAX_QUEUE} websocket_max_size={WS_MAX_SIZE}")
|
||||
@ -1230,14 +1520,47 @@ class ESP32LiveKitBridge:
|
||||
session.tts_active = False
|
||||
session.tts_thinking = False
|
||||
session.tts_stream_id += 1
|
||||
cleanup_tasks: list[asyncio.Task[Any]] = []
|
||||
current_task = asyncio.current_task()
|
||||
room_connect_pending = (
|
||||
session.room_connect_task is not None
|
||||
and not session.room_connect_task.done()
|
||||
)
|
||||
|
||||
if (
|
||||
not room_connect_pending
|
||||
and session.agent_dispatch_task is not None
|
||||
and not session.agent_dispatch_task.done()
|
||||
):
|
||||
session.agent_dispatch_task.cancel()
|
||||
if session.agent_dispatch_task is not current_task:
|
||||
cleanup_tasks.append(session.agent_dispatch_task)
|
||||
session.agent_dispatch_task = None
|
||||
|
||||
cleanup_tasks.extend(self._cancel_forwarding_tracks(session))
|
||||
|
||||
if session.tts_idle_task is not None:
|
||||
session.tts_idle_task.cancel()
|
||||
if session.tts_idle_task is not current_task:
|
||||
cleanup_tasks.append(session.tts_idle_task)
|
||||
session.tts_idle_task = None
|
||||
self._cancel_tts_display_task(session)
|
||||
try:
|
||||
await session.room.disconnect()
|
||||
except Exception as exc:
|
||||
print(f"断开 LiveKit 房间失败: room={session.room_name} error={exc}")
|
||||
if session.tts_display_task is not None:
|
||||
session.tts_display_task.cancel()
|
||||
if session.tts_display_task is not current_task:
|
||||
cleanup_tasks.append(session.tts_display_task)
|
||||
session.tts_display_task = None
|
||||
|
||||
for task in list(session.background_tasks):
|
||||
if task is current_task or task.done():
|
||||
continue
|
||||
task.cancel()
|
||||
cleanup_tasks.append(task)
|
||||
|
||||
if cleanup_tasks:
|
||||
await asyncio.gather(*cleanup_tasks, return_exceptions=True)
|
||||
|
||||
if not room_connect_pending:
|
||||
await self._disconnect_room_quietly(session, "session_close")
|
||||
|
||||
async def forward_audio_to_esp32(
|
||||
self,
|
||||
@ -1407,11 +1730,20 @@ class ESP32LiveKitBridge:
|
||||
except Exception as exc:
|
||||
print(f"音频流处理错误: {exc}")
|
||||
finally:
|
||||
close_stream = getattr(audio_stream, "aclose", None) or getattr(audio_stream, "close", None)
|
||||
if close_stream is not None:
|
||||
try:
|
||||
result = close_stream()
|
||||
if result is not None and hasattr(result, "__await__"):
|
||||
await result
|
||||
except Exception as exc:
|
||||
print(f"关闭 LiveKit 音频流失败: {exc}")
|
||||
print("🎧 TTS 音频结束")
|
||||
task = session.forwarding_tracks.get(track_sid)
|
||||
current_task = asyncio.current_task()
|
||||
if task is current_task:
|
||||
session.forwarding_tracks.pop(track_sid, None)
|
||||
session.forwarding_track_participants.pop(track_sid, None)
|
||||
if stream_id == session.tts_stream_id and session.tts_idle_task is not None:
|
||||
session.tts_idle_task.cancel()
|
||||
session.tts_idle_task = None
|
||||
@ -1473,9 +1805,14 @@ class ESP32LiveKitBridge:
|
||||
f"已发送 server hello: device={device_id} room={session.room_name} "
|
||||
f"audio={OUTPUT_SAMPLE_RATE}Hz/{OUTPUT_FRAME_DURATION_MS}ms"
|
||||
)
|
||||
asyncio.create_task(self._run_emotion_test_sequence(session))
|
||||
self._create_session_task(
|
||||
session,
|
||||
self._run_emotion_test_sequence(session),
|
||||
"emotion 测试序列",
|
||||
)
|
||||
|
||||
room_connect_task = asyncio.create_task(self._connect_session_room(session))
|
||||
session.room_connect_task = room_connect_task
|
||||
room_connect_task.add_done_callback(
|
||||
lambda task: self._track_room_connect_task(session, task)
|
||||
)
|
||||
@ -1551,6 +1888,8 @@ class ESP32LiveKitBridge:
|
||||
abort_reason = reason if isinstance(reason, str) and reason else "button_abort"
|
||||
print(f"处理 ESP32 打断请求: reason={abort_reason}")
|
||||
await self._abort_tts(session, abort_reason)
|
||||
elif msg_type == "mcp":
|
||||
await self._publish_mcp_message(session, data)
|
||||
elif msg_type == "vision" and data.get("state") == "frame":
|
||||
await self._publish_vision_frame(session, data)
|
||||
except json.JSONDecodeError:
|
||||
@ -1561,10 +1900,6 @@ class ESP32LiveKitBridge:
|
||||
self._log_exception("WebSocket 其他错误", exc)
|
||||
finally:
|
||||
print(f"ESP32 断开连接: device={device_id} room={session.room_name}")
|
||||
if room_connect_task is not None and not room_connect_task.done():
|
||||
room_connect_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await room_connect_task
|
||||
await self._close_session(session)
|
||||
self.device_sessions.pop(device_id, None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user