Beaver/beaver_project
6
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: implement channel runtime connectors

Browse Source
This commit is contained in:
steven_li
2026-06-03 16:22:44 +08:00
parent ee972441f5
commit c3d84b904a
105 changed files with 15621 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

520
app-instance/backend/beaver/interfaces/web/app.py
View File

@ -19,6 +19,18 @@ from typing import Any
from beaver.engine.providers.registry import PROVIDERS, find_by_name
from beaver.foundation.config import default_config_path, load_config
from beaver.foundation.events import ChannelIdentity, InboundMessage
from beaver.interfaces.channels.runtime import ChannelRuntime
from beaver.interfaces.channels.connections import (
ChannelConnectionStore,
ChannelConnectorRegistry,
ConnectorSidecarClient,
CredentialStore,
FeishuConnector,
MessageDedupeStore,
TelegramConnector,
WeixinConnector,
)
from beaver.foundation.models import CronExecutionResult, CronRunRecord
from beaver.integrations.mcp import MCPConnectionManager
from beaver.services.agent_service import NOTIFICATION_SESSION_ID, AgentService
@ -53,6 +65,16 @@ from .schemas import (
WebErrorResponse,
WebAgentConfigRequest,
WebAgentConfigResponse,
WebChannelConfigRequest,
WebChannelConfigResponse,
WebChannelConnectionCreateRequest,
WebChannelConnectionResponse,
WebChannelConnectionUpdateRequest,
WebChannelValidationResponse,
WebConnectorBridgeEventRequest,
WebConnectorBridgeEventResponse,
WebConnectorSessionCreateRequest,
WebConnectorSessionResponse,
WebProviderConfigRequest,
WebProviderConfigResponse,
WebStatusResponse,
@ -60,7 +82,7 @@ from .schemas import (
try:
from fastapi import FastAPI, File, Form, Header, HTTPException, Request, UploadFile, WebSocket, WebSocketDisconnect
from fastapi.responses import Response
from fastapi.responses import JSONResponse, Response
except ModuleNotFoundError: # pragma: no cover - fallback for skeleton-only environments
def File(default: Any = None) -> Any: # type: ignore[override]
return default
@ -94,6 +116,11 @@ except ModuleNotFoundError: # pragma: no cover - fallback for skeleton-only env
self.media_type = media_type
self.headers = headers or {}
class JSONResponse(Response): # type: ignore[override]
def __init__(self, content: Any, status_code: int = 200) -> None:
super().__init__(json.dumps(content).encode("utf-8"), media_type="application/json")
self.status_code = status_code
class WebSocketDisconnect(Exception):
"""Fallback websocket disconnect exception."""
@ -183,7 +210,9 @@ async def _app_lifespan(
owns_service = manage_service_lifecycle if manage_service_lifecycle is not None else service is None
app.state.agent_service = attached_service
app.state.cron_service = _build_cron_service(attached_service) if owns_service else None
app.state.channel_runtime = None
started = False
channel_runtime: ChannelRuntime | None = None
if owns_service:
try:
await attached_service.start()
@ -200,6 +229,29 @@ async def _app_lifespan(
else:
attached_service.close()
raise
try:
loaded = attached_service.create_loop().boot()
app.state.channel_connection_workspace = loaded.workspace
connector_registry = _build_channel_connector_registry(loaded.workspace)
app.state.channel_connector_registry = connector_registry
connection_channels = await connector_registry.materialize_channel_configs()
runtime_channels = dict(loaded.config.channels)
runtime_channels.update(connection_channels)
channel_runtime = ChannelRuntime(
service=attached_service,
workspace=loaded.workspace,
channels=runtime_channels,
)
app.state.channel_runtime = channel_runtime
await channel_runtime.start()
except BaseException:
if owns_service and started:
with suppress(BaseException):
await attached_service.shutdown(
timeout_seconds=shutdown_timeout_seconds,
force=shutdown_force,
)
raise
worker: SkillLearningWorker | None = None
worker_task = None
worker_config = SkillLearningWorkerConfig.from_env()
@ -216,6 +268,10 @@ async def _app_lifespan(
try:
yield
finally:
runtime = getattr(app.state, "channel_runtime", None)
if isinstance(runtime, ChannelRuntime):
with suppress(BaseException):
await runtime.stop()
cron_service = getattr(app.state, "cron_service", None)
if isinstance(cron_service, CronService):
cron_service.stop()
@ -283,6 +339,118 @@ def get_cron_service(request: Request) -> CronService:
return service
def get_channel_runtime(request: Request) -> ChannelRuntime:
runtime = getattr(request.app.state, "channel_runtime", None)
if not isinstance(runtime, ChannelRuntime):
raise HTTPException(status_code=503, detail="Channel runtime is not running")
return runtime
def _connection_state_dir(workspace: Path) -> Path:
return Path(workspace) / "state" / "channel_connections"
def _channel_connection_workspace(request: Request) -> Path:
workspace = getattr(request.app.state, "channel_connection_workspace", None)
if workspace is not None:
return Path(workspace)
return Path(get_agent_service(request).loader.workspace)
def _message_dedupe_store(workspace: Path) -> MessageDedupeStore:
return MessageDedupeStore(_connection_state_dir(workspace) / "message_dedupe.json")
def _bridge_token() -> str:
return os.getenv("BEAVER_BRIDGE_TOKEN", "")
def _build_channel_connector_registry(workspace: Path) -> ChannelConnectorRegistry:
state_dir = _connection_state_dir(workspace)
connection_store = ChannelConnectionStore(state_dir / "connections.json")
credential_store = CredentialStore(state_dir / "credentials.json")
registry = ChannelConnectorRegistry(connection_store=connection_store, credential_store=credential_store)
registry.register(
TelegramConnector(
connection_store=connection_store,
credential_store=credential_store,
)
)
sidecar_base_url = os.getenv("EXTERNAL_CONNECTOR_BASE_URL", "http://external-connector:8787")
sidecar_token = os.getenv("EXTERNAL_CONNECTOR_TOKEN", "")
sidecar_client = ConnectorSidecarClient(base_url=sidecar_base_url, token=sidecar_token)
registry.register(
WeixinConnector(
connection_store=connection_store,
credential_store=credential_store,
sidecar_client=sidecar_client,
sidecar_base_url=sidecar_base_url,
)
)
registry.register(
FeishuConnector(
connection_store=connection_store,
credential_store=credential_store,
sidecar_client=sidecar_client,
sidecar_base_url=sidecar_base_url,
)
)
return registry
def get_channel_connector_registry(request: Request) -> ChannelConnectorRegistry:
registry = getattr(request.app.state, "channel_connector_registry", None)
if isinstance(registry, ChannelConnectorRegistry):
return registry
workspace = getattr(request.app.state, "channel_connection_workspace", None)
if workspace is None:
raise RuntimeError("Channel connector registry unavailable before service boot")
registry = _build_channel_connector_registry(workspace)
request.app.state.channel_connector_registry = registry
return registry
def _connection_response_view(connection: Any) -> dict[str, Any]:
view = connection.to_dict()
view.pop("credentials_ref", None)
view.pop("connector_ref", None)
view.pop("pairing_session_id", None)
return view
def _normalize_connection_config(config: dict[str, Any] | None) -> dict[str, Any]:
if not isinstance(config, dict):
return {}
return {
_camel_to_snake_text(str(key)): value
for key, value in config.items()
if str(key).strip()
}
def _camel_to_snake_text(value: str) -> str:
result: list[str] = []
for char in value.strip():
if char.isupper() and result:
result.append("_")
result.append(char.lower())
return "".join(result)
def _self_restart_enabled() -> bool:
return os.getenv("BEAVER_ENABLE_SELF_RESTART", "1").strip() not in {"0", "false", "False"}
def _schedule_self_restart(delay_seconds: float = 0.75) -> None:
import threading
def _exit_later() -> None:
time.sleep(delay_seconds)
os._exit(0)
threading.Thread(target=_exit_later, daemon=True).start()
def create_app(
*,
workspace: str | Path | None = None,
@ -380,10 +548,330 @@ def create_app(
"temperature": agent_service.profile.temperature,
"max_tool_iterations": agent_service.profile.max_tool_iterations,
"providers": providers_status,
"channels": [{"name": "web", "enabled": True}],
"channels": get_channel_runtime(request).statuses(),
"runtime_controls": {"self_restart": _self_restart_enabled()},
"cron": cron_service.status(),
}
@app.get("/api/channels")
async def list_channels(request: Request) -> list[dict[str, Any]]:
return get_channel_runtime(request).statuses()
@app.get("/api/channel-connectors")
async def list_channel_connectors(request: Request) -> list[dict[str, str]]:
return get_channel_connector_registry(request).connectors()
@app.get("/api/channel-connections")
async def list_channel_connections(request: Request) -> list[dict[str, Any]]:
registry = get_channel_connector_registry(request)
return [_connection_response_view(connection) for connection in registry.connection_store.list()]
@app.post("/api/channel-connections", response_model=WebChannelConnectionResponse)
async def create_channel_connection(
request: Request,
payload: WebChannelConnectionCreateRequest,
) -> WebChannelConnectionResponse:
registry = get_channel_connector_registry(request)
kind = _clean_text(payload.kind)
mode = _clean_text(payload.mode)
if not kind:
raise HTTPException(status_code=400, detail="Connection kind is required")
if not mode:
raise HTTPException(status_code=400, detail="Connection mode is required")
secrets_payload = payload.secrets or {}
secrets = {key: value for key, value in secrets_payload.items() if value}
credentials_ref = registry.credential_store.put(kind=kind, values=secrets) if secrets else None
connection = registry.connection_store.create(
kind=kind,
mode=mode,
display_name=_clean_text(payload.display_name) or kind,
account_id=_clean_text(payload.account_id) or "",
owner_user_id=_clean_text(payload.owner_user_id) or None,
auth_type=_clean_text(payload.auth_type) or "token",
credentials_ref=credentials_ref,
runtime_config=_normalize_connection_config(payload.config),
)
return WebChannelConnectionResponse(
connection=_connection_response_view(connection),
credentials=registry.credential_store.redacted(credentials_ref),
)
@app.patch("/api/channel-connections/{connection_id}", response_model=WebChannelConnectionResponse)
async def update_channel_connection(
connection_id: str,
request: Request,
payload: WebChannelConnectionUpdateRequest,
) -> WebChannelConnectionResponse:
registry = get_channel_connector_registry(request)
try:
connection = registry.connection_store.get(connection_id)
except KeyError:
raise HTTPException(status_code=404, detail="Channel connection not found")
if payload.display_name is not None:
connection.display_name = _clean_text(payload.display_name) or connection.display_name
if payload.account_id is not None:
connection.account_id = _clean_text(payload.account_id) or connection.account_id
if payload.config is not None:
connection.runtime_config = _normalize_connection_config(payload.config)
if payload.secrets:
secrets = {key: value for key, value in payload.secrets.items() if value}
if secrets:
# TODO: add credential GC when connection updates credentials.
connection.credentials_ref = registry.credential_store.put(kind=connection.kind, values=secrets)
connection = registry.connection_store.update(connection)
return WebChannelConnectionResponse(
connection=_connection_response_view(connection),
credentials=registry.credential_store.redacted(connection.credentials_ref),
)
@app.get("/api/channel-connections/{connection_id}", response_model=WebChannelConnectionResponse)
async def get_channel_connection(connection_id: str, request: Request) -> WebChannelConnectionResponse:
registry = get_channel_connector_registry(request)
try:
connection = registry.connection_store.get(connection_id)
except KeyError:
raise HTTPException(status_code=404, detail="Channel connection not found")
return WebChannelConnectionResponse(
connection=_connection_response_view(connection),
credentials=registry.credential_store.redacted(connection.credentials_ref),
)
@app.post("/api/channel-connections/{connection_id}/validate", response_model=WebChannelValidationResponse)
async def validate_channel_connection(connection_id: str, request: Request) -> WebChannelValidationResponse:
registry = get_channel_connector_registry(request)
try:
result = await registry.validate(connection_id)
connection = registry.connection_store.get(connection_id)
except KeyError:
raise HTTPException(status_code=404, detail="Channel connection not found")
return WebChannelValidationResponse(
ok=result.ok,
status=result.status,
account_id=result.account_id,
display_name=result.display_name,
error=result.error,
metadata=result.metadata,
connection=_connection_response_view(connection),
)
@app.post("/api/channel-connections/{connection_id}/revoke", response_model=WebChannelConnectionResponse)
async def revoke_channel_connection(connection_id: str, request: Request) -> WebChannelConnectionResponse:
registry = get_channel_connector_registry(request)
try:
await registry.revoke(connection_id)
connection = registry.connection_store.get(connection_id)
except KeyError:
raise HTTPException(status_code=404, detail="Channel connection not found")
return WebChannelConnectionResponse(connection=_connection_response_view(connection), credentials={})
@app.post("/api/channel-connector-sessions", response_model=WebConnectorSessionResponse)
async def start_channel_connector_session(
request: Request,
payload: WebConnectorSessionCreateRequest,
) -> WebConnectorSessionResponse:
registry = get_channel_connector_registry(request)
kind = _clean_text(payload.kind)
try:
connector = registry.connector_for_kind(kind)
except KeyError:
raise HTTPException(status_code=404, detail="Connector not found")
start_session = getattr(connector, "start_session", None)
if start_session is None:
raise HTTPException(status_code=400, detail="Connector does not support sessions")
view = await start_session(
display_name=_clean_text(payload.display_name) or kind,
owner_user_id=_clean_text(payload.owner_user_id) or None,
options=payload.options,
)
connection_id = _clean_text(view.get("connectionId"))
connection_view = None
if connection_id:
connection_view = _connection_response_view(registry.connection_store.get(connection_id))
return WebConnectorSessionResponse(session=view, connection=connection_view)
@app.get("/api/channel-connector-sessions/{session_id}", response_model=WebConnectorSessionResponse)
async def get_channel_connector_session(session_id: str, request: Request) -> WebConnectorSessionResponse:
registry = get_channel_connector_registry(request)
connection = next(
(item for item in registry.connection_store.list() if item.pairing_session_id == session_id),
None,
)
if connection is None:
raise HTTPException(status_code=404, detail="Connector session not found")
connector = registry.connector_for_kind(connection.kind)
poll_session = getattr(connector, "poll_session", None)
if poll_session is None:
raise HTTPException(status_code=400, detail="Connector does not support sessions")
view = await poll_session(session_id)
connection = registry.connection_store.get(connection.connection_id)
if connection.status == "connected":
runtime = get_channel_runtime(request)
config = (await registry.materialize_channel_configs())[connection.channel_id]
await runtime.add_channel(connection.channel_id, config)
return WebConnectorSessionResponse(session=view, connection=_connection_response_view(connection))
@app.post("/api/channel-connector-bridge/events", response_model=WebConnectorBridgeEventResponse)
async def accept_connector_bridge_event(
request: Request,
payload: WebConnectorBridgeEventRequest,
authorization: str | None = Header(default=None),
) -> Any:
expected = _bridge_token()
if not expected or authorization != f"Bearer {expected}":
raise HTTPException(status_code=401, detail="Invalid connector bridge token")
registry = get_channel_connector_registry(request)
try:
connection = registry.connection_store.get(payload.connection_id)
except KeyError:
raise HTTPException(status_code=404, detail="Channel connection not found")
if connection.status == "revoked":
raise HTTPException(status_code=404, detail="Channel connection not found")
store = _message_dedupe_store(_channel_connection_workspace(request))
begin = store.begin(
connection_id=payload.connection_id,
event_id=payload.event_id,
delivery_attempt=payload.delivery_attempt,
)
if not begin.should_process:
body = WebConnectorBridgeEventResponse(
accepted=begin.http_status == 200,
duplicate=True,
pending=begin.http_status == 409,
retryAfterSeconds=begin.retry_after_seconds,
).model_dump(by_alias=True)
return JSONResponse(status_code=begin.http_status, content=body)
runtime = get_channel_runtime(request)
identity = ChannelIdentity(
channel_id=payload.channel_id,
kind=payload.kind,
account_id=payload.account_id,
peer_id=payload.peer_id,
thread_id=payload.thread_id,
peer_type=payload.peer_type,
user_id=payload.user_id,
message_id=payload.message_id,
)
inbound = InboundMessage(
channel=payload.channel_id,
content=payload.content,
content_type=payload.message_type,
channel_identity=identity,
user_id=payload.user_id,
message_id=payload.message_id,
metadata=dict(payload.metadata),
)
result = await runtime.accept_inbound(inbound)
if result.accepted or result.duplicate:
store.complete(begin.dedupe_key, message_id=payload.message_id)
else:
store.fail(begin.dedupe_key, error=result.error or "runtime rejected bridge event")
return WebConnectorBridgeEventResponse(
accepted=result.accepted,
duplicate=result.duplicate,
pending=result.pending,
)
@app.get("/api/channels/{channel_id}/config")
async def get_channel_config(channel_id: str, request: Request) -> dict[str, Any]:
agent_service = get_agent_service(request)
config_path = agent_service.loader.config.config_path or default_config_path(workspace=agent_service.loader.workspace)
raw = _read_config_json(config_path)
channel = _ensure_dict(raw, "channels").get(channel_id)
if not isinstance(channel, dict):
raise HTTPException(status_code=404, detail="Channel not found")
return _channel_config_view(channel_id, channel)
@app.post("/api/channels/{channel_id}/config", response_model=WebChannelConfigResponse)
async def update_channel_config(
channel_id: str,
request: Request,
payload: WebChannelConfigRequest,
) -> WebChannelConfigResponse:
if not _clean_text(channel_id):
raise HTTPException(status_code=400, detail="Channel id is required")
kind = _clean_text(payload.kind)
mode = _clean_text(payload.mode)
if not kind:
raise HTTPException(status_code=400, detail="Channel kind is required")
if not mode:
raise HTTPException(status_code=400, detail="Channel mode is required")
agent_service = get_agent_service(request)
config_path = agent_service.loader.config.config_path or default_config_path(workspace=agent_service.loader.workspace)
raw = _read_config_json(config_path)
channels = _ensure_dict(raw, "channels")
current = channels.get(channel_id) if isinstance(channels.get(channel_id), dict) else {}
current_secrets = current.get("secrets") if isinstance(current.get("secrets"), dict) else {}
next_secrets = dict(current_secrets)
for key, value in (payload.secrets or {}).items():
cleaned_key = _clean_text(key)
cleaned_value = _clean_text(value)
if not cleaned_key or not cleaned_value:
continue
next_secrets[cleaned_key] = cleaned_value
channel_payload: dict[str, Any] = {
"enabled": bool(payload.enabled),
"kind": kind,
"mode": mode,
"accountId": _clean_text(payload.account_id) or "",
"displayName": _clean_text(payload.display_name) or channel_id,
"config": payload.config or {},
"secrets": next_secrets,
}
channels[channel_id] = channel_payload
_write_config_json(config_path, raw)
_reload_agent_config(agent_service, config_path)
return WebChannelConfigResponse(
ok=True,
channel_id=channel_id,
restart_required=True,
channel=_channel_config_view(channel_id, channel_payload),
)
@app.get("/api/channels/{channel_id}/events")
async def list_channel_events(channel_id: str, request: Request, limit: int = 100) -> list[dict[str, Any]]:
return get_channel_runtime(request).recent_events(channel_id, limit=limit)
@app.post("/api/channels/{channel_id}/webhook")
async def post_channel_webhook(channel_id: str, request: Request) -> JSONResponse:
runtime = get_channel_runtime(request)
adapter = runtime.adapters.get(channel_id)
if adapter is None or not hasattr(adapter, "handle_webhook_payload"):
raise HTTPException(status_code=404, detail="Webhook channel not found")
payload = await request.json()
if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="Webhook payload must be a JSON object")
result = await adapter.handle_webhook_payload(payload) # type: ignore[attr-defined]
status_code = 202 if result.get("pending") else 200
return JSONResponse(result, status_code=status_code)
@app.websocket("/api/channels/{channel_id}/ws")
async def channel_websocket(websocket: WebSocket, channel_id: str) -> None:
runtime = getattr(websocket.app.state, "channel_runtime", None)
if not isinstance(runtime, ChannelRuntime):
await websocket.accept()
await websocket.send_json({"type": "error", "error": "Channel runtime is not running"})
await websocket.close(code=1011)
return
adapter = runtime.adapters.get(channel_id)
if adapter is None or not hasattr(adapter, "handle_websocket"):
await websocket.accept()
await websocket.send_json({"type": "error", "error": "WebSocket channel not found"})
await websocket.close(code=1008)
return
await adapter.handle_websocket(websocket) # type: ignore[attr-defined]
@app.post("/api/runtime/restart")
async def restart_runtime() -> JSONResponse:
if not _self_restart_enabled():
raise HTTPException(status_code=403, detail="Self restart is disabled")
_schedule_self_restart()
return JSONResponse({"ok": True, "restarting": True}, status_code=202)
@app.post("/api/auth/login")
async def auth_login(request: Request, payload: dict[str, Any]) -> dict[str, Any]:
username = _clean_text(payload.get("username"))
@ -3011,6 +3499,25 @@ def _mask_secret(value: str | None) -> str:
return f"{secret[:4]}••••{secret[-4:]}"
def _channel_config_view(channel_id: str, data: dict[str, Any]) -> dict[str, Any]:
secrets_payload = data.get("secrets") if isinstance(data.get("secrets"), dict) else {}
config_payload = data.get("config") if isinstance(data.get("config"), dict) else {}
return {
"channel_id": channel_id,
"enabled": bool(data.get("enabled")),
"kind": _clean_text(data.get("kind")) or "",
"mode": _clean_text(data.get("mode")) or "webhook",
"account_id": _clean_text(data.get("accountId") or data.get("account_id")) or "",
"display_name": _clean_text(data.get("displayName") or data.get("display_name")) or channel_id,
"config": dict(config_payload),
"secrets": {
str(key): _mask_secret(str(value) if value is not None else None)
for key, value in secrets_payload.items()
if str(key).strip()
},
}
def _read_config_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
@ -3082,7 +3589,14 @@ def _reload_agent_config(agent_service: AgentService, config_path: Path) -> None
old_manager = getattr(loaded, "mcp_manager", None)
if old_manager is not None:
async def _close_old_manager() -> None:
await old_manager.close()
try:
await old_manager.close()
except Exception:
# MCP transports may own anyio cancel scopes created by a
# previous request task. Config reload must not leak that
# cleanup failure as an unhandled background exception or
# knock the app out of running mode.
pass
try:
running_loop = asyncio.get_running_loop()