diff --git a/.env.example b/.env.example index 2b5917b..2df2fff 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,12 @@ NANO_MODEL=openai/gpt-5 NANO_API_KEY=sk-xxxxxxxx NANO_API_BASE= +# Per-instance Beaver backend config. In Docker app-instance this should point +# to the mounted single-user sandbox config, not to frontend env. +BEAVER_HOME=/root/.beaver +BEAVER_CONFIG_PATH=/root/.beaver/config.json +BEAVER_WORKSPACE=/root/.beaver/workspace + # Must be reachable from app-instance containers. NANO_AUTHZ_URL=http://nano-authz-service:19090 NANO_OUTLOOK_MCP_URL= diff --git a/app-instance/Dockerfile b/app-instance/Dockerfile index fe1672b..7de3bc4 100644 --- a/app-instance/Dockerfile +++ b/app-instance/Dockerfile @@ -36,7 +36,10 @@ ENV DEBIAN_FRONTEND=noninteractive \ APP_PUBLIC_PORT=8080 \ APP_FRONTEND_PORT=3000 \ APP_BACKEND_PORT=18080 \ - NANOBOT_AUTH_FILE=/root/.nanobot/web_auth_users.json \ + BEAVER_HOME=/root/.beaver \ + BEAVER_CONFIG_PATH=/root/.beaver/config.json \ + BEAVER_WORKSPACE=/root/.beaver/workspace \ + NANOBOT_AUTH_FILE=/root/.beaver/web_auth_users.json \ PORT=3000 \ HOSTNAME=127.0.0.1 @@ -58,25 +61,10 @@ RUN apt-get update && \ WORKDIR /opt/app/backend -COPY backend/pyproject.toml backend/README.md backend/LICENSE ./ -RUN mkdir -p nanobot bridge && touch nanobot/__init__.py && \ - uv pip install --system --no-cache . - -COPY backend/nanobot/ ./nanobot/ -COPY backend/bridge/ ./bridge/ -COPY backend/third_party/swarms/ ./third_party/swarms/ +COPY backend/pyproject.toml backend/README.md ./ +COPY backend/beaver/ ./beaver/ RUN uv pip install --system --no-cache . -WORKDIR /opt/app/backend/bridge -RUN --mount=type=cache,target=/root/.npm \ - git config --global url."https://github.com/".insteadOf "ssh://git@github.com/" && \ - git config --global url."https://github.com/".insteadOf "git@github.com:" && \ - npm config set registry "https://registry.npmjs.org" && \ - npm config set fetch-retries "${NPM_FETCH_RETRIES}" && \ - npm config set fetch-retry-mintimeout "${NPM_FETCH_RETRY_MIN_TIMEOUT}" && \ - npm config set fetch-retry-maxtimeout "${NPM_FETCH_RETRY_MAX_TIMEOUT}" && \ - npm install && npm run build - WORKDIR /opt/app/frontend COPY --from=frontend-builder /build/frontend/next.config.js ./ COPY --from=frontend-builder /build/frontend/public ./public @@ -89,7 +77,7 @@ COPY nginx.conf /opt/app/nginx.conf COPY entrypoint.sh /opt/app/entrypoint.sh RUN chmod +x /opt/app/entrypoint.sh && \ - mkdir -p /var/lib/nginx/body /root/.nanobot/workspace + mkdir -p /var/lib/nginx/body /root/.beaver/workspace EXPOSE 8080 diff --git a/app-instance/README.md b/app-instance/README.md index 73ae06c..149f7d6 100644 --- a/app-instance/README.md +++ b/app-instance/README.md @@ -106,17 +106,33 @@ runtime/instances// ```text runtime/instances// -└── nanobot-home +└── beaver-home ├── config.json ├── web_auth_users.json └── workspace/ ``` +这个目录是单用户 sandbox 的配置与数据边界。容器内会把它挂到： + +```text +/root/.beaver/ +``` + +并设置： + +```text +BEAVER_CONFIG_PATH=/root/.beaver/config.json +BEAVER_WORKSPACE=/root/.beaver/workspace +``` + +所以模型 `provider/api_key/api_base/model` 配一次即可，Web / channel 请求不需要、也不应该携带 API Key。 + ## 当前状态 这层已经支持： - 统一镜像构建 +- 镜像内安装并启动新的 `beaver` 后端 - 实例创建 - 实例删除 - 实例列表 diff --git a/app-instance/backend/beaver/engine/loader.py b/app-instance/backend/beaver/engine/loader.py index 89e0fcb..3f7b0e6 100644 --- a/app-instance/backend/beaver/engine/loader.py +++ b/app-instance/backend/beaver/engine/loader.py @@ -2,17 +2,27 @@ from __future__ import annotations +import os from dataclasses import dataclass, field from pathlib import Path from typing import Callable from beaver.engine.context import ContextBuilder from beaver.engine.session import SessionManager +from beaver.foundation.config import BeaverConfig, load_config from beaver.memory.curated.store import MemoryStore from beaver.services.memory_service import MemoryService from beaver.skills import SkillAssembler, SkillsLoader -from beaver.tools import ObjectBackedTool, ToolExecutor, ToolRegistry -from beaver.tools.builtins import EchoTool, MemoryTool, SessionSearchTool, SkillViewTool +from beaver.tools import ObjectBackedTool, ToolAssembler, ToolExecutor, ToolRegistry +from beaver.tools.builtins import ( + EchoTool, + ListDirectoryTool, + MemoryTool, + ReadFileTool, + SearchFilesTool, + SessionSearchTool, + SkillViewTool, +) @dataclass(slots=True) @@ -27,6 +37,7 @@ class EngineLoadResult: """ workspace: Path + config: BeaverConfig = field(default_factory=BeaverConfig) tools: list[str] = field(default_factory=list) skills: list[str] = field(default_factory=list) memory_stores: list[str] = field(default_factory=list) @@ -35,6 +46,7 @@ class EngineLoadResult: curated_memory_store: MemoryStore | None = None memory_service: MemoryService | None = None tool_registry: ToolRegistry | None = None + tool_assembler: ToolAssembler | None = None tool_executor: ToolExecutor | None = None context_builder: ContextBuilder | None = None skills_loader: SkillsLoader | None = None @@ -89,19 +101,26 @@ class EngineLoader: self, *, workspace: str | Path | None = None, + config_path: str | Path | None = None, + config: BeaverConfig | None = None, session_manager: SessionManager | None = None, curated_memory_store: MemoryStore | None = None, memory_service: MemoryService | None = None, tool_registry: ToolRegistry | None = None, + tool_assembler: ToolAssembler | None = None, context_builder: ContextBuilder | None = None, skills_loader: SkillsLoader | None = None, skill_assembler: SkillAssembler | None = None, ) -> None: - self.workspace = Path(workspace or Path.cwd()) + self.config = config or load_config(workspace=workspace, config_path=config_path) + configured_workspace = self.config.agents_defaults.workspace + env_workspace = os.getenv("BEAVER_WORKSPACE") + self.workspace = Path(workspace or configured_workspace or env_workspace or Path.cwd()) self._session_manager = session_manager self._curated_memory_store = curated_memory_store self._memory_service = memory_service self._tool_registry = tool_registry + self._tool_assembler = tool_assembler self._context_builder = context_builder self._skills_loader = skills_loader self._skill_assembler = skill_assembler @@ -127,15 +146,20 @@ class EngineLoader: ObjectBackedTool(MemoryTool(store=memory_service.get_store())), ObjectBackedTool(SkillViewTool(loader=skills_loader)), ObjectBackedTool(SessionSearchTool(db=session_manager)), + ObjectBackedTool(ListDirectoryTool()), + ObjectBackedTool(ReadFileTool()), + ObjectBackedTool(SearchFilesTool()), ] ) context_builder = self._context_builder or ContextBuilder() + tool_assembler = self._tool_assembler or ToolAssembler() tool_executor = ToolExecutor(tool_registry) skill_assembler = self._skill_assembler or SkillAssembler(skills_loader) result = EngineLoadResult( workspace=workspace, + config=self.config, tools=[spec.name for spec in tool_registry.list_specs()], skills=[record.name for record in skills_loader.list_skills(filter_unavailable=False)], memory_stores=["curated"], @@ -144,6 +168,7 @@ class EngineLoader: curated_memory_store=memory_service.get_store(), memory_service=memory_service, tool_registry=tool_registry, + tool_assembler=tool_assembler, tool_executor=tool_executor, context_builder=context_builder, skills_loader=skills_loader, diff --git a/app-instance/backend/beaver/engine/loop.py b/app-instance/backend/beaver/engine/loop.py index 66afaea..af6bdd6 100644 --- a/app-instance/backend/beaver/engine/loop.py +++ b/app-instance/backend/beaver/engine/loop.py @@ -272,12 +272,24 @@ class AgentLoop: memory_service = self._require_loaded("memory_service") context_builder = self._require_loaded("context_builder") tool_registry = self._require_loaded("tool_registry") + tool_assembler = self._require_loaded("tool_assembler") tool_executor = self._require_loaded("tool_executor") + skills_loader = self._require_loaded("skills_loader") skill_assembler = self._require_loaded("skill_assembler") + config = loaded.config + configured_provider = config.resolve_provider_target(model=model, provider_name=provider_name) + resolved_session_id = session_id or uuid4().hex resolved_run_id = uuid4().hex - resolved_model = model or self.profile.default_model + resolved_model = configured_provider.get("model") or self.profile.default_model + resolved_provider_name = configured_provider.get("provider_name") or provider_name + resolved_api_key = api_key or configured_provider.get("api_key") + resolved_api_base = api_base or configured_provider.get("api_base") + resolved_extra_headers = extra_headers or configured_provider.get("extra_headers") + resolved_request_timeout_seconds = configured_provider.get("request_timeout_seconds") + resolved_embedding_model = embedding_model or config.default_embedding_model + resolved_embedding_target = embedding_target or config.resolve_embedding_target() resolved_max_tokens = max_tokens or self.profile.max_tokens resolved_temperature = self.profile.temperature if temperature is None else temperature resolved_max_tool_iterations = ( @@ -316,20 +328,21 @@ class AgentLoop: user_message_recorded = False iterations = 0 final_usage: dict[str, Any] = {} - final_provider_name: str | None = provider_name + final_provider_name: str | None = resolved_provider_name final_model: str | None = resolved_model try: bundle = provider_bundle or make_provider_bundle( model=resolved_model, - provider_name=provider_name, - api_key=api_key, - api_base=api_base, - extra_headers=extra_headers, + provider_name=resolved_provider_name, + api_key=resolved_api_key, + api_base=resolved_api_base, + request_timeout_seconds=resolved_request_timeout_seconds, + extra_headers=resolved_extra_headers, routing=routing, fallback_target=fallback_target, auxiliary_target=auxiliary_target, - embedding_target=embedding_target, - embedding_model=embedding_model or "text-embedding-v4", + embedding_target=resolved_embedding_target, + embedding_model=resolved_embedding_model, ) skill_selector_provider = bundle.auxiliary_provider or bundle.main_provider skill_selector_model = ( @@ -364,6 +377,32 @@ class AgentLoop: user_id=user_id, ) + selected_tool_specs = await tool_assembler.assemble( + task_description=task, + registry=tool_registry, + skills_loader=skills_loader, + activated_skills=assembled_skills.activated_skills, + embedding_runtime=bundle.embedding_runtime, + top_k=10, + ) + tool_schemas = tool_registry.export_selected_provider_schemas(selected_tool_specs) + session_manager.append_message( + resolved_session_id, + run_id=resolved_run_id, + role="system", + event_type="tool_selection_snapshotted", + event_payload={ + "tools": [spec.to_mcp_descriptor() for spec in selected_tool_specs], + "tool_names": [spec.name for spec in selected_tool_specs], + }, + content=", ".join(spec.name for spec in selected_tool_specs) or None, + context_visible=False, + source=source, + title=title, + model=resolved_model, + user_id=user_id, + ) + build_input = ContextBuildInput( base_system_prompt=self.profile.system_prompt, history=session_manager.get_history(resolved_session_id), @@ -412,7 +451,6 @@ class AgentLoop: provider = bundle.main_provider messages = list(context_result.messages) - tool_schemas = tool_registry.export_provider_schemas() tool_context = ToolContext( workspace=str(loaded.workspace), session_id=resolved_session_id, diff --git a/app-instance/backend/beaver/engine/providers/litellm.py b/app-instance/backend/beaver/engine/providers/litellm.py index 2e4b76b..24b2bb7 100644 --- a/app-instance/backend/beaver/engine/providers/litellm.py +++ b/app-instance/backend/beaver/engine/providers/litellm.py @@ -8,7 +8,7 @@ import os from typing import Any from .base import LLMProvider, LLMResponse, ToolCallRequest -from .registry import find_by_model, find_gateway +from .registry import find_by_model, find_by_name, find_gateway from .runtime import ProviderRoutingConfig try: # pragma: no cover - optional dependency @@ -58,7 +58,11 @@ class LiteLLMProvider(LLMProvider): if not api_key: return {} - spec = self._gateway or find_by_model(model) + spec = self._gateway + if spec is None and self.provider_name: + spec = find_by_name(self.provider_name) + if spec is None: + spec = find_by_model(model) if spec is None or not spec.env_key: return {} overrides: dict[str, str] = {spec.env_key: api_key} @@ -97,6 +101,15 @@ class LiteLLMProvider(LLMProvider): if prefix and not resolved.startswith(f"{prefix}/"): resolved = f"{prefix}/{resolved}" return resolved + if self.provider_name: + spec = find_by_name(self.provider_name) + if spec is not None and not spec.is_gateway and not spec.is_local: + resolved = model + if spec.litellm_prefix and not any(resolved.startswith(prefix) for prefix in spec.skip_prefixes): + resolved = f"{spec.litellm_prefix}/{resolved}" + elif spec.name == "openai" and "/" not in resolved: + resolved = f"openai/{resolved}" + return resolved spec = find_by_model(model) if spec and spec.litellm_prefix: if not any(model.startswith(prefix) for prefix in spec.skip_prefixes): diff --git a/app-instance/backend/beaver/foundation/config/__init__.py b/app-instance/backend/beaver/foundation/config/__init__.py index 42cd1cb..6d2c228 100644 --- a/app-instance/backend/beaver/foundation/config/__init__.py +++ b/app-instance/backend/beaver/foundation/config/__init__.py @@ -1,2 +1,13 @@ """Configuration models and loaders.""" +from .loader import default_config_path, load_config +from .schema import AgentDefaultsConfig, BeaverConfig, EmbeddingConfig, ProviderConfig + +__all__ = [ + "AgentDefaultsConfig", + "BeaverConfig", + "EmbeddingConfig", + "ProviderConfig", + "default_config_path", + "load_config", +] diff --git a/app-instance/backend/beaver/foundation/config/loader.py b/app-instance/backend/beaver/foundation/config/loader.py new file mode 100644 index 0000000..8f179e6 --- /dev/null +++ b/app-instance/backend/beaver/foundation/config/loader.py @@ -0,0 +1,127 @@ +"""Config loader for per-sandbox Beaver runtime settings.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +from .schema import AgentDefaultsConfig, BeaverConfig, EmbeddingConfig, ProviderConfig + + +def default_config_path(*, workspace: str | Path | None = None) -> Path: + """Resolve the default config path for a single-user sandbox instance. + + Priority: + 1. `BEAVER_CONFIG_PATH` + 2. `NANOBOT_CONFIG_PATH` for compatibility during migration + 3. `BEAVER_HOME/config.json` + 4. `NANOBOT_HOME/config.json` for migration compatibility + 5. `/.beaver/config.json` + 6. `./.beaver/config.json` + """ + + explicit = os.getenv("BEAVER_CONFIG_PATH") or os.getenv("NANOBOT_CONFIG_PATH") + if explicit: + return Path(explicit).expanduser() + + beaver_home = os.getenv("BEAVER_HOME") + if beaver_home: + return Path(beaver_home).expanduser() / "config.json" + + nanobot_home = os.getenv("NANOBOT_HOME") + if nanobot_home: + return Path(nanobot_home).expanduser() / "config.json" + + root = Path(workspace).expanduser() if workspace is not None else Path.cwd() + return root / ".beaver" / "config.json" + + +def load_config( + *, + workspace: str | Path | None = None, + config_path: str | Path | None = None, +) -> BeaverConfig: + """Load backend config; missing config is treated as an empty config.""" + + path = Path(config_path).expanduser() if config_path is not None else default_config_path(workspace=workspace) + if not path.exists(): + return BeaverConfig(config_path=path) + + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError(f"Beaver config must be a JSON object: {path}") + + return BeaverConfig( + agents_defaults=_parse_agent_defaults(data), + providers=_parse_providers(data.get("providers")), + embedding=_parse_embedding(data), + config_path=path, + ) + + +def _parse_agent_defaults(data: dict[str, Any]) -> AgentDefaultsConfig: + agents = _as_dict(data.get("agents")) + defaults = _as_dict(agents.get("defaults")) + return AgentDefaultsConfig( + workspace=_string(defaults.get("workspace") or data.get("workspace")), + model=_string(defaults.get("model") or data.get("model")), + provider=_string(defaults.get("provider") or data.get("provider")), + embedding_model=_string(defaults.get("embeddingModel") or defaults.get("embedding_model") or data.get("embeddingModel")), + ) + + +def _parse_providers(raw: Any) -> dict[str, ProviderConfig]: + providers: dict[str, ProviderConfig] = {} + for name, payload in _as_dict(raw).items(): + if not isinstance(payload, dict): + continue + providers[str(name)] = ProviderConfig( + api_key=_string(payload.get("apiKey") or payload.get("api_key")), + api_base=_string(payload.get("apiBase") or payload.get("api_base") or payload.get("baseUrl") or payload.get("base_url")), + extra_headers=_string_dict(payload.get("extraHeaders") or payload.get("extra_headers") or payload.get("headers")), + request_timeout_seconds=_float( + payload.get("requestTimeoutSeconds") + or payload.get("request_timeout_seconds") + or payload.get("timeout") + ), + ) + return providers + + +def _parse_embedding(data: dict[str, Any]) -> EmbeddingConfig: + raw = _as_dict(data.get("embedding") or data.get("embeddings")) + return EmbeddingConfig( + provider=_string(raw.get("provider") or raw.get("provider_name")), + model=_string(raw.get("model") or data.get("embeddingModel") or data.get("embedding_model")), + api_key=_string(raw.get("apiKey") or raw.get("api_key")), + api_base=_string(raw.get("apiBase") or raw.get("api_base") or raw.get("baseUrl") or raw.get("base_url")), + extra_headers=_string_dict(raw.get("extraHeaders") or raw.get("extra_headers") or raw.get("headers")), + request_timeout_seconds=_float( + raw.get("requestTimeoutSeconds") or raw.get("request_timeout_seconds") or raw.get("timeout") + ), + ) + + +def _as_dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _string(value: Any) -> str | None: + if value is None: + return None + value = str(value).strip() + return value or None + + +def _string_dict(value: Any) -> dict[str, str]: + if not isinstance(value, dict): + return {} + return {str(key): str(item) for key, item in value.items() if item is not None} + + +def _float(value: Any) -> float | None: + if value in (None, ""): + return None + return float(value) diff --git a/app-instance/backend/beaver/foundation/config/schema.py b/app-instance/backend/beaver/foundation/config/schema.py new file mode 100644 index 0000000..7344982 --- /dev/null +++ b/app-instance/backend/beaver/foundation/config/schema.py @@ -0,0 +1,136 @@ +"""Runtime configuration schema for Beaver sandbox instances.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass(slots=True) +class ProviderConfig: + """One configured LLM provider profile.""" + + api_key: str | None = None + api_base: str | None = None + extra_headers: dict[str, str] = field(default_factory=dict) + request_timeout_seconds: float | None = None + + +@dataclass(slots=True) +class AgentDefaultsConfig: + """Default agent settings for this sandbox instance.""" + + workspace: str | None = None + model: str | None = None + provider: str | None = None + embedding_model: str | None = None + + +@dataclass(slots=True) +class EmbeddingConfig: + """Optional dedicated embedding model settings.""" + + provider: str | None = None + model: str | None = None + api_key: str | None = None + api_base: str | None = None + extra_headers: dict[str, str] = field(default_factory=dict) + request_timeout_seconds: float | None = None + + +@dataclass(slots=True) +class BeaverConfig: + """Config loaded once per backend sandbox instance.""" + + agents_defaults: AgentDefaultsConfig = field(default_factory=AgentDefaultsConfig) + providers: dict[str, ProviderConfig] = field(default_factory=dict) + embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig) + config_path: Path | None = None + + @property + def default_model(self) -> str | None: + return _clean(self.agents_defaults.model) + + @property + def default_embedding_model(self) -> str: + return _clean(self.embedding.model) or _clean(self.agents_defaults.embedding_model) or "text-embedding-v4" + + def resolve_provider_target( + self, + *, + model: str | None = None, + provider_name: str | None = None, + ) -> dict[str, Any]: + """Resolve model/provider credentials from instance config. + + Request-level model/provider overrides are allowed, but credentials are still + read from backend config, not from Web/channel payloads. + """ + + resolved_model = _clean(model) or self.default_model + resolved_provider = _clean(provider_name) or self._infer_provider(resolved_model) + provider_cfg = self.providers.get(resolved_provider or "") if resolved_provider else None + payload: dict[str, Any] = { + "model": resolved_model, + "provider_name": resolved_provider, + } + if provider_cfg is not None: + payload.update( + { + "api_key": provider_cfg.api_key, + "api_base": provider_cfg.api_base, + "extra_headers": dict(provider_cfg.extra_headers), + "request_timeout_seconds": provider_cfg.request_timeout_seconds, + } + ) + return {key: value for key, value in payload.items() if value not in (None, "", {})} + + def resolve_embedding_target(self) -> dict[str, Any] | None: + """Return an explicit embedding target when configured.""" + + has_explicit_embedding = any( + [ + _clean(self.embedding.provider), + _clean(self.embedding.api_key), + _clean(self.embedding.api_base), + self.embedding.extra_headers, + self.embedding.request_timeout_seconds is not None, + ] + ) + if not has_explicit_embedding: + return None + + provider_cfg = self.providers.get(_clean(self.embedding.provider) or "") + payload: dict[str, Any] = { + "provider": _clean(self.embedding.provider), + "model": self.default_embedding_model, + "api_key": _clean(self.embedding.api_key) or (provider_cfg.api_key if provider_cfg else None), + "api_base": _clean(self.embedding.api_base) or (provider_cfg.api_base if provider_cfg else None), + "extra_headers": self.embedding.extra_headers or (dict(provider_cfg.extra_headers) if provider_cfg else {}), + "request_timeout_seconds": self.embedding.request_timeout_seconds + or (provider_cfg.request_timeout_seconds if provider_cfg else None), + } + return {key: value for key, value in payload.items() if value not in (None, "", {})} + + def _infer_provider(self, model: str | None) -> str | None: + configured_provider = _clean(self.agents_defaults.provider) + if configured_provider: + return configured_provider + + if model and "/" in model: + prefix = model.split("/", 1)[0] + if prefix in self.providers: + return prefix + + if len(self.providers) == 1: + return next(iter(self.providers)) + return None + + +def _clean(value: str | None) -> str | None: + if value is None: + return None + value = str(value).strip() + return value or None + diff --git a/app-instance/backend/beaver/foundation/embedding.py b/app-instance/backend/beaver/foundation/embedding.py new file mode 100644 index 0000000..bb2b39d --- /dev/null +++ b/app-instance/backend/beaver/foundation/embedding.py @@ -0,0 +1,205 @@ +"""Shared embedding-based semantic retrieval utilities.""" + +from __future__ import annotations + +import asyncio +import json +import math +import os +from typing import Any +from urllib import request + + +class EmbeddingRetriever: + """Use an OpenAI-compatible embeddings API to rank lightweight candidates.""" + + def __init__( + self, + *, + api_key_env: str = "OPENAI_API_KEY", + api_base_env: str = "OPENAI_API_BASE", + model: str = "text-embedding-v4", + timeout_seconds: float = 20.0, + ) -> None: + self.api_key_env = api_key_env + self.api_base_env = api_base_env + self.model = model + self.timeout_seconds = timeout_seconds + + async def retrieve( + self, + *, + query: str, + candidates: list[dict[str, str]], + top_k: int, + api_key: str | None = None, + api_base: str | None = None, + model: str | None = None, + extra_headers: dict[str, str] | None = None, + timeout_seconds: float | None = None, + fallback_top_k: int | None = None, + ) -> list[dict[str, str]]: + """Return candidates ordered by embedding similarity. + + If embedding config is missing or the request fails, return the original + candidate order. This keeps retrieval non-blocking for the main run. + """ + + if not candidates or top_k <= 0: + return [] + + fallback = self._fallback_candidates(candidates, fallback_top_k=fallback_top_k) + resolved_api_key = api_key or os.getenv(self.api_key_env) + resolved_api_base = api_base or os.getenv(self.api_base_env) + if not resolved_api_key or not resolved_api_base: + return fallback + + try: + query_embedding = await self._embed_texts( + api_key=resolved_api_key, + api_base=resolved_api_base, + texts=[query], + model=model or self.model, + extra_headers=extra_headers, + timeout_seconds=timeout_seconds, + ) + candidate_embeddings = await self._embed_texts( + api_key=resolved_api_key, + api_base=resolved_api_base, + texts=[self._candidate_text(item) for item in candidates], + model=model or self.model, + extra_headers=extra_headers, + timeout_seconds=timeout_seconds, + ) + except Exception: + return fallback + + if not query_embedding or not query_embedding[0] or len(candidate_embeddings) != len(candidates): + return fallback + + query_vector = query_embedding[0] + scored: list[tuple[float, dict[str, str]]] = [] + for candidate, vector in zip(candidates, candidate_embeddings, strict=False): + if vector: + scored.append((self._cosine_similarity(query_vector, vector), candidate)) + + scored.sort(key=lambda item: item[0], reverse=True) + return [item[1] for item in scored[:top_k]] + + async def _embed_texts( + self, + *, + api_key: str, + api_base: str, + texts: list[str], + model: str, + extra_headers: dict[str, str] | None = None, + timeout_seconds: float | None = None, + ) -> list[list[float]]: + all_vectors: list[list[float]] = [] + endpoint = self._normalize_embeddings_endpoint(api_base) + for start in range(0, len(texts), 10): + batch = texts[start:start + 10] + payload = await self._post_embeddings( + endpoint=endpoint, + api_key=api_key, + model=model, + texts=batch, + extra_headers=extra_headers, + timeout_seconds=timeout_seconds, + ) + embeddings = payload.get("data") or [] + embeddings = sorted(embeddings, key=lambda item: item.get("index", 0)) + all_vectors.extend([list(item.get("embedding") or []) for item in embeddings]) + return all_vectors + + async def _post_embeddings( + self, + *, + endpoint: str, + api_key: str, + model: str, + texts: list[str], + extra_headers: dict[str, str] | None = None, + timeout_seconds: float | None = None, + ) -> dict[str, Any]: + return await asyncio.to_thread( + self._post_embeddings_sync, + endpoint=endpoint, + api_key=api_key, + model=model, + texts=texts, + extra_headers=extra_headers, + timeout_seconds=timeout_seconds, + ) + + def _post_embeddings_sync( + self, + *, + endpoint: str, + api_key: str, + model: str, + texts: list[str], + extra_headers: dict[str, str] | None = None, + timeout_seconds: float | None = None, + ) -> dict[str, Any]: + body = json.dumps( + { + "model": model, + "input": texts if len(texts) > 1 else texts[0], + "encoding_format": "float", + } + ).encode("utf-8") + req = request.Request( + endpoint, + data=body, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + **(extra_headers or {}), + }, + method="POST", + ) + with request.urlopen(req, timeout=timeout_seconds or self.timeout_seconds) as response: + return json.loads(response.read().decode("utf-8")) + + @staticmethod + def _fallback_candidates( + candidates: list[dict[str, str]], + *, + fallback_top_k: int | None, + ) -> list[dict[str, str]]: + if fallback_top_k is None: + return list(candidates) + if fallback_top_k <= 0: + return [] + return candidates[:fallback_top_k] + + @staticmethod + def _candidate_text(candidate: dict[str, str]) -> str: + parts = [ + (candidate.get("name") or "").strip(), + (candidate.get("description") or "").strip(), + (candidate.get("input_schema") or "").strip(), + ] + return "\n".join(part for part in parts if part) + + @staticmethod + def _normalize_embeddings_endpoint(api_base: str) -> str: + base = api_base.rstrip("/") + if base.endswith("/embeddings"): + return base + if base.endswith("/v1"): + return f"{base}/embeddings" + return f"{base}/v1/embeddings" + + @staticmethod + def _cosine_similarity(left: list[float], right: list[float]) -> float: + if not left or not right or len(left) != len(right): + return -1.0 + dot = sum(a * b for a, b in zip(left, right, strict=False)) + left_norm = math.sqrt(sum(a * a for a in left)) + right_norm = math.sqrt(sum(b * b for b in right)) + if left_norm == 0 or right_norm == 0: + return -1.0 + return dot / (left_norm * right_norm) diff --git a/app-instance/backend/beaver/interfaces/channels/__init__.py b/app-instance/backend/beaver/interfaces/channels/__init__.py index bf9247a..97f4a30 100644 --- a/app-instance/backend/beaver/interfaces/channels/__init__.py +++ b/app-instance/backend/beaver/interfaces/channels/__init__.py @@ -1,2 +1,7 @@ """Channel interfaces.""" +from .base import ChannelAdapter +from .manager import ChannelManager +from .memory import MemoryChannelAdapter + +__all__ = ["ChannelAdapter", "ChannelManager", "MemoryChannelAdapter"] diff --git a/app-instance/backend/beaver/interfaces/channels/base.py b/app-instance/backend/beaver/interfaces/channels/base.py new file mode 100644 index 0000000..40e3767 --- /dev/null +++ b/app-instance/backend/beaver/interfaces/channels/base.py @@ -0,0 +1,24 @@ +"""Channel adapter contracts for gateway-facing integrations.""" + +from __future__ import annotations + +from typing import Protocol + +from beaver.foundation.events import MessageBus, OutboundMessage + + +class ChannelAdapter(Protocol): + """Minimal contract every gateway channel must implement.""" + + name: str + bus: MessageBus + + async def start(self) -> None: + """Prepare the channel before messages are routed.""" + + async def stop(self) -> None: + """Stop accepting/routing channel messages.""" + + async def send(self, message: OutboundMessage) -> None: + """Deliver an outbound message to the concrete channel.""" + diff --git a/app-instance/backend/beaver/interfaces/channels/manager.py b/app-instance/backend/beaver/interfaces/channels/manager.py new file mode 100644 index 0000000..438191b --- /dev/null +++ b/app-instance/backend/beaver/interfaces/channels/manager.py @@ -0,0 +1,76 @@ +"""Channel manager for routing gateway outbound messages.""" + +from __future__ import annotations + +import asyncio +from contextlib import suppress + +from beaver.foundation.events import MessageBus, OutboundMessage + +from .base import ChannelAdapter + + +class ChannelManager: + """Start/stop channel adapters and dispatch outbound messages to them.""" + + def __init__(self, bus: MessageBus) -> None: + self.bus = bus + self.channels: dict[str, ChannelAdapter] = {} + self.undeliverable: list[OutboundMessage] = [] + self.started = False + + def register(self, channel: ChannelAdapter) -> None: + if self.started: + raise RuntimeError("Cannot register channels after ChannelManager.start()") + if channel.name in self.channels: + raise ValueError(f"Channel already registered: {channel.name}") + if channel.bus is not self.bus: + raise ValueError("Channel must share the same MessageBus as ChannelManager") + self.channels[channel.name] = channel + + async def start(self) -> None: + started: list[ChannelAdapter] = [] + try: + for channel in self.channels.values(): + await channel.start() + started.append(channel) + except BaseException: + for channel in reversed(started): + with suppress(BaseException): + await channel.stop() + raise + else: + self.started = True + + async def stop(self) -> None: + errors: list[BaseException] = [] + for channel in reversed(tuple(self.channels.values())): + try: + await channel.stop() + except Exception as exc: # pragma: no cover - defensive cleanup path + errors.append(exc) + self.started = False + if errors: + raise RuntimeError(f"Failed to stop {len(errors)} channel(s)") from errors[0] + + async def dispatch_outbound(self, stop_event: asyncio.Event) -> None: + """Route bus outbound messages until stopped and the queue is drained.""" + + while True: + if stop_event.is_set() and self.bus.outbound_size == 0: + break + + try: + message = await asyncio.wait_for(self.bus.consume_outbound(), timeout=0.25) + except asyncio.TimeoutError: + continue + + channel = self.channels.get(message.channel) + if channel is None: + self.undeliverable.append(message) + continue + + try: + await channel.send(message) + except Exception: # pragma: no cover - defensive channel isolation + self.undeliverable.append(message) diff --git a/app-instance/backend/beaver/interfaces/channels/memory.py b/app-instance/backend/beaver/interfaces/channels/memory.py new file mode 100644 index 0000000..a70a553 --- /dev/null +++ b/app-instance/backend/beaver/interfaces/channels/memory.py @@ -0,0 +1,57 @@ +"""In-memory channel adapter for tests and local gateway embedding.""" + +from __future__ import annotations + +from typing import Any + +from beaver.foundation.events import InboundMessage, MessageBus, OutboundMessage + + +class MemoryChannelAdapter: + """A local channel that stores outbound messages in memory.""" + + def __init__(self, bus: MessageBus, *, name: str = "memory") -> None: + self.name = name + self.bus = bus + self.started = False + self.sent_messages: list[OutboundMessage] = [] + + async def start(self) -> None: + self.started = True + + async def stop(self) -> None: + self.started = False + + async def send(self, message: OutboundMessage) -> None: + self.sent_messages.append(message) + + async def publish_text( + self, + content: str, + *, + session_id: str | None = None, + user_id: str | None = None, + title: str | None = None, + execution_context: str | None = None, + model: str | None = None, + provider_name: str | None = None, + embedding_model: str | None = None, + metadata: dict[str, Any] | None = None, + ) -> InboundMessage: + """Publish a text message from this channel into the shared bus.""" + + message = InboundMessage( + channel=self.name, + content=content, + session_id=session_id, + user_id=user_id, + title=title, + execution_context=execution_context, + model=model, + provider_name=provider_name, + embedding_model=embedding_model, + metadata=metadata or {}, + ) + await self.bus.publish_inbound(message) + return message + diff --git a/app-instance/backend/beaver/interfaces/cli/main.py b/app-instance/backend/beaver/interfaces/cli/main.py index 40f5ebc..5a7d131 100644 --- a/app-instance/backend/beaver/interfaces/cli/main.py +++ b/app-instance/backend/beaver/interfaces/cli/main.py @@ -35,6 +35,7 @@ app = typer.Typer(help="Beaver backend CLI") if hasattr(typer, "Typer") else typ def run( message: str | None = typer.Option(None, "--message", "-m", help="Run one direct Beaver request."), workspace: str | None = typer.Option(None, "--workspace", help="Workspace root for this run."), + config: str | None = typer.Option(None, "--config", help="Backend config path for this run."), ) -> None: """Thin CLI wrapper around AgentService. @@ -44,7 +45,7 @@ def run( 3. 打印结果 """ - service = AgentService(workspace=workspace) + service = AgentService(workspace=workspace, config_path=config) if not message: service.create_loop() typer.echo("Beaver engine booted.") diff --git a/app-instance/backend/beaver/interfaces/gateway/main.py b/app-instance/backend/beaver/interfaces/gateway/main.py index 1ceac26..4901aef 100644 --- a/app-instance/backend/beaver/interfaces/gateway/main.py +++ b/app-instance/backend/beaver/interfaces/gateway/main.py @@ -1,41 +1,39 @@ """Gateway entrypoint for Beaver. -当前阶段先不扩 bus / channels adapter，只做最小消息桥接： +当前阶段只做最小 gateway 宿主与 channel adapter 桥接： 1. 启动时托管 `AgentService.start()` 2. 常驻消费 `MessageBus.inbound` -3. 调 `service.submit_direct(...)` +3. 调 `service.handle_inbound_message(...)` 4. 将结果写回 `MessageBus.outbound` -5. 退出时走 `AgentService.shutdown()` +5. 如果配置了 channel adapters，则由 `ChannelManager` 分发 outbound +6. 退出时走 `AgentService.shutdown()` """ from __future__ import annotations import asyncio +from collections.abc import Sequence +from contextlib import suppress from pathlib import Path -from beaver.foundation.events import InboundMessage, MessageBus, OutboundMessage +from beaver.foundation.events import InboundMessage, MessageBus +from beaver.interfaces.channels import ChannelAdapter, ChannelManager from beaver.services.agent_service import AgentService -async def _publish_bridge_error( - bus: MessageBus, - inbound: InboundMessage, +async def _cleanup_owned_service( + service: AgentService, *, - detail: str, - finish_reason: str = "error", + timeout_seconds: float | None, + force: bool, ) -> None: - """把 bridge 处理失败转换成结构化 outbound 错误消息。""" + """Best-effort cleanup for service startup failures or cancellations.""" - await bus.publish_outbound( - OutboundMessage( - message_id=inbound.message_id, - channel=inbound.channel, - session_id=inbound.session_id, - content=detail, - finish_reason=finish_reason, - metadata={"error": detail, "inbound_metadata": dict(inbound.metadata)}, - ) - ) + with suppress(BaseException): + if service.is_running: + await service.shutdown(timeout_seconds=timeout_seconds, force=force) + else: + service.close() async def _flush_pending_inbound(bus: MessageBus, *, reason: str) -> None: @@ -46,11 +44,17 @@ async def _flush_pending_inbound(bus: MessageBus, *, reason: str) -> None: pending = bus.inbound.get_nowait() except asyncio.QueueEmpty: break - await _publish_bridge_error(bus, pending, detail=reason, finish_reason="stopped") + await bus.publish_outbound( + AgentService.build_outbound_error( + pending, + detail=reason, + finish_reason="stopped", + ) + ) -async def _await_bridge_shutdown(task: asyncio.Task[None], *, timeout_seconds: float = 1.0) -> None: - """等待 bridge 退出；超时则取消，避免 shutdown 被桥接层反向卡死。""" +async def _await_task_shutdown(task: asyncio.Task[None], *, timeout_seconds: float = 1.0) -> None: + """等待后台任务退出；超时则取消，避免 shutdown 被反向卡死。""" try: await asyncio.wait_for(task, timeout=timeout_seconds) @@ -85,53 +89,28 @@ async def _bridge_inbound_to_runtime( continue try: - result = await service.submit_direct( - inbound.content, - session_id=inbound.session_id, - source=f"gateway:{inbound.channel}", - user_id=inbound.user_id, - title=inbound.title, - execution_context=inbound.execution_context, - model=inbound.model, - provider_name=inbound.provider_name, - embedding_model=inbound.embedding_model, - ) + outbound = await service.handle_inbound_message(inbound) except asyncio.CancelledError: - await _publish_bridge_error( - bus, - inbound, - detail="Gateway stopped before completing the inbound message", - finish_reason="cancelled", - ) - raise - except Exception as exc: # pragma: no cover - defensive bridge path - await _publish_bridge_error( - bus, - inbound, - detail=str(exc), - ) - else: await bus.publish_outbound( - OutboundMessage( - message_id=inbound.message_id, - channel=inbound.channel, - session_id=result.session_id, - run_id=result.run_id, - content=result.output_text, - finish_reason=result.finish_reason, - provider_name=result.provider_name, - model=result.model, - usage=dict(result.usage), - metadata={"inbound_metadata": dict(inbound.metadata)}, + AgentService.build_outbound_error( + inbound, + detail="Gateway stopped before completing the inbound message", + finish_reason="cancelled", ) ) + raise + else: + await bus.publish_outbound(outbound) async def run_gateway( *, workspace: str | Path | None = None, + config_path: str | Path | None = None, service: AgentService | None = None, bus: MessageBus | None = None, + channels: Sequence[ChannelAdapter] | None = None, + channel_manager: ChannelManager | None = None, manage_service_lifecycle: bool | None = None, stop_event: asyncio.Event | None = None, shutdown_timeout_seconds: float | None = 5.0, @@ -142,19 +121,41 @@ async def run_gateway( 默认 ownership 语义： - 未传 `service`：gateway 自己创建并接管其 lifecycle - 传入外部 `service`：默认只使用，不自动 start/shutdown + - `channel_manager` 和 `channels` 二选一，避免隐式修改外部 manager """ - attached_service = service or AgentService(workspace=workspace) - attached_bus = bus or MessageBus() + attached_service = service or AgentService(workspace=workspace, config_path=config_path) + if channel_manager is not None and channels is not None: + raise ValueError("Pass either channel_manager or channels, not both") + if bus is not None: + attached_bus = bus + elif channel_manager is not None: + attached_bus = channel_manager.bus + else: + attached_bus = MessageBus() + attached_channel_manager = channel_manager + if attached_channel_manager is not None and attached_channel_manager.bus is not attached_bus: + raise ValueError("Injected channel_manager must share the gateway MessageBus") + if attached_channel_manager is None and channels is not None: + attached_channel_manager = ChannelManager(attached_bus) + if attached_channel_manager is not None and channels is not None: + for channel in channels: + attached_channel_manager.register(channel) + owns_service = manage_service_lifecycle if manage_service_lifecycle is not None else service is None owned_stop_event = stop_event or asyncio.Event() started = False + channels_started = False if owns_service: try: await attached_service.start() started = True - except Exception: - attached_service.close() + except BaseException: + await _cleanup_owned_service( + attached_service, + timeout_seconds=shutdown_timeout_seconds, + force=shutdown_force, + ) raise if not attached_service.is_running: @@ -163,7 +164,25 @@ async def run_gateway( "or allow the gateway to manage its lifecycle." ) + if attached_channel_manager is not None: + try: + await attached_channel_manager.start() + channels_started = True + except BaseException: + if owns_service and started: + await _cleanup_owned_service( + attached_service, + timeout_seconds=shutdown_timeout_seconds, + force=shutdown_force, + ) + raise + bridge_task = asyncio.create_task(_bridge_inbound_to_runtime(attached_service, attached_bus, owned_stop_event)) + dispatch_task: asyncio.Task[None] | None = None + dispatch_stop_event = asyncio.Event() + if attached_channel_manager is not None: + dispatch_task = asyncio.create_task(attached_channel_manager.dispatch_outbound(dispatch_stop_event)) + try: await owned_stop_event.wait() finally: @@ -175,9 +194,14 @@ async def run_gateway( force=shutdown_force, ) finally: - await _await_bridge_shutdown(bridge_task) + await _await_task_shutdown(bridge_task) else: - await _await_bridge_shutdown(bridge_task) + await _await_task_shutdown(bridge_task) + if dispatch_task is not None: + dispatch_stop_event.set() + await _await_task_shutdown(dispatch_task) + if attached_channel_manager is not None and channels_started: + await attached_channel_manager.stop() def main() -> None: diff --git a/app-instance/backend/beaver/interfaces/web/app.py b/app-instance/backend/beaver/interfaces/web/app.py index c3a14a5..069ffe6 100644 --- a/app-instance/backend/beaver/interfaces/web/app.py +++ b/app-instance/backend/beaver/interfaces/web/app.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from pathlib import Path from types import SimpleNamespace from typing import Any @@ -56,6 +56,7 @@ async def _app_lifespan( app: FastAPI, *, workspace: str | Path | None, + config_path: str | Path | None, service: AgentService | None, manage_service_lifecycle: bool | None, shutdown_timeout_seconds: float | None, @@ -63,7 +64,7 @@ async def _app_lifespan( ) -> AsyncIterator[None]: """把 Web app 接到 AgentService lifecycle 上。""" - attached_service = service or AgentService(workspace=workspace) + attached_service = service or AgentService(workspace=workspace, config_path=config_path) owns_service = manage_service_lifecycle if manage_service_lifecycle is not None else service is None app.state.agent_service = attached_service started = False @@ -71,8 +72,15 @@ async def _app_lifespan( try: await attached_service.start() started = True - except Exception: - attached_service.close() + except BaseException: + with suppress(BaseException): + if attached_service.is_running: + await attached_service.shutdown( + timeout_seconds=shutdown_timeout_seconds, + force=shutdown_force, + ) + else: + attached_service.close() raise try: yield @@ -87,6 +95,7 @@ async def _app_lifespan( def create_app( *, workspace: str | Path | None = None, + config_path: str | Path | None = None, service: AgentService | None = None, manage_service_lifecycle: bool | None = None, shutdown_timeout_seconds: float | None = 5.0, @@ -106,6 +115,7 @@ def create_app( lifespan=lambda fastapi_app: _app_lifespan( fastapi_app, workspace=workspace, + config_path=config_path, service=service, manage_service_lifecycle=manage_service_lifecycle, shutdown_timeout_seconds=shutdown_timeout_seconds, diff --git a/app-instance/backend/beaver/services/__init__.py b/app-instance/backend/beaver/services/__init__.py index 69a53e3..1c77d6c 100644 --- a/app-instance/backend/beaver/services/__init__.py +++ b/app-instance/backend/beaver/services/__init__.py @@ -1,6 +1,15 @@ """Application services for Beaver.""" -from .agent_service import AgentService -from .memory_service import MemoryService - __all__ = ["AgentService", "MemoryService"] + + +def __getattr__(name: str): + if name == "AgentService": + from .agent_service import AgentService + + return AgentService + if name == "MemoryService": + from .memory_service import MemoryService + + return MemoryService + raise AttributeError(name) diff --git a/app-instance/backend/beaver/services/agent_service.py b/app-instance/backend/beaver/services/agent_service.py index 0c479e9..f407618 100644 --- a/app-instance/backend/beaver/services/agent_service.py +++ b/app-instance/backend/beaver/services/agent_service.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any from beaver.engine import AgentLoop, AgentProfile, AgentRunResult, EngineLoader +from beaver.foundation.events import InboundMessage, OutboundMessage class AgentService: @@ -36,11 +37,12 @@ class AgentService: self, *, workspace: str | Path | None = None, + config_path: str | Path | None = None, profile: AgentProfile | None = None, loader: EngineLoader | None = None, ) -> None: self.profile = profile or AgentProfile() - self.loader = loader or EngineLoader(workspace=workspace) + self.loader = loader or EngineLoader(workspace=workspace, config_path=config_path) self._loop: AgentLoop | None = None self._run_task: asyncio.Task[None] | None = None @@ -189,6 +191,60 @@ class AgentService: loop = self.create_loop() return await loop.submit_direct(message, **kwargs) + async def handle_inbound_message(self, inbound: InboundMessage) -> OutboundMessage: + """把 bus inbound 映射成标准 runtime 调用，并返回结构化 outbound。""" + + try: + result = await self.submit_direct( + inbound.content, + session_id=inbound.session_id, + source=f"gateway:{inbound.channel}", + user_id=inbound.user_id, + title=inbound.title, + execution_context=inbound.execution_context, + model=inbound.model, + provider_name=inbound.provider_name, + embedding_model=inbound.embedding_model, + ) + except Exception as exc: + return self.build_outbound_error(inbound, detail=str(exc)) + return self.build_outbound_message(inbound, result) + + @staticmethod + def build_outbound_message(inbound: InboundMessage, result: AgentRunResult) -> OutboundMessage: + """把一次 runtime 正常结果转成 bus outbound。""" + + return OutboundMessage( + message_id=inbound.message_id, + channel=inbound.channel, + session_id=result.session_id, + run_id=result.run_id, + content=result.output_text, + finish_reason=result.finish_reason, + provider_name=result.provider_name, + model=result.model, + usage=dict(result.usage), + metadata={"inbound_metadata": dict(inbound.metadata)}, + ) + + @staticmethod + def build_outbound_error( + inbound: InboundMessage, + *, + detail: str, + finish_reason: str = "error", + ) -> OutboundMessage: + """把 inbound 处理失败转换成结构化 outbound 错误消息。""" + + return OutboundMessage( + message_id=inbound.message_id, + channel=inbound.channel, + session_id=inbound.session_id, + content=detail, + finish_reason=finish_reason, + metadata={"error": detail, "inbound_metadata": dict(inbound.metadata)}, + ) + def run_direct( self, message: str, diff --git a/app-instance/backend/beaver/skills/__init__.py b/app-instance/backend/beaver/skills/__init__.py index b90112c..2d8d583 100644 --- a/app-instance/backend/beaver/skills/__init__.py +++ b/app-instance/backend/beaver/skills/__init__.py @@ -1,7 +1,12 @@ -"""Skill system for Beaver.""" +"""Skill system for Beaver. -from .assembler import SkillAssembler, SkillAssemblyResult, SkillEmbeddingRetriever -from .catalog import SkillRecord, SkillsLoader +顶层包保持 lazy export，避免只导入 catalog/loader 时顺带拉起 +SkillAssembler -> provider -> litellm 这条重依赖链。 +""" + +from __future__ import annotations + +from typing import Any __all__ = [ "SkillAssembler", @@ -10,3 +15,22 @@ __all__ = [ "SkillRecord", "SkillsLoader", ] + + +def __getattr__(name: str) -> Any: + if name in {"SkillAssembler", "SkillAssemblyResult", "SkillEmbeddingRetriever"}: + from .assembler import SkillAssembler, SkillAssemblyResult, SkillEmbeddingRetriever + + return { + "SkillAssembler": SkillAssembler, + "SkillAssemblyResult": SkillAssemblyResult, + "SkillEmbeddingRetriever": SkillEmbeddingRetriever, + }[name] + if name in {"SkillRecord", "SkillsLoader"}: + from .catalog import SkillRecord, SkillsLoader + + return { + "SkillRecord": SkillRecord, + "SkillsLoader": SkillsLoader, + }[name] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/app-instance/backend/beaver/skills/assembler/embedding_retriever.py b/app-instance/backend/beaver/skills/assembler/embedding_retriever.py index a7d74e7..55e734a 100644 --- a/app-instance/backend/beaver/skills/assembler/embedding_retriever.py +++ b/app-instance/backend/beaver/skills/assembler/embedding_retriever.py @@ -1,188 +1,9 @@ -"""Embedding-based skill candidate retrieval. - -当前实现使用 OpenAI-compatible `/v1/embeddings` 接口调用 -阿里云百炼 `text-embedding-v4` 做最小语义召回： -1. 复用当前 provider 的 `api_key/api_base` -2. 先用 embedding 相似度召回一小批候选 -3. 再交给上层 LLM selector 做最终技能选择 -""" +"""Embedding-based skill candidate retrieval.""" from __future__ import annotations -import asyncio -import math -import os -import json -from urllib import request -from typing import Any +from beaver.foundation.embedding import EmbeddingRetriever -class SkillEmbeddingRetriever: +class SkillEmbeddingRetriever(EmbeddingRetriever): """用 OpenAI-compatible embeddings API 为 skill 选择做候选召回。""" - - def __init__( - self, - *, - api_key_env: str = "OPENAI_API_KEY", - api_base_env: str = "OPENAI_API_BASE", - model: str = "text-embedding-v4", - timeout_seconds: float = 20.0, - ) -> None: - self.api_key_env = api_key_env - self.api_base_env = api_base_env - self.model = model - self.timeout_seconds = timeout_seconds - - async def retrieve( - self, - *, - query: str, - candidates: list[dict[str, str]], - top_k: int = 12, - api_key: str | None = None, - api_base: str | None = None, - model: str | None = None, - ) -> list[dict[str, str]]: - """按 embedding 相似度召回 top-k 候选。 - - 如果没有可用的 API Key / base URL，或者 embedding 调用失败， - 当前阶段先退回到“全部候选交给 LLM selector”。 - """ - - if not candidates: - return [] - - resolved_api_key = api_key or os.getenv(self.api_key_env) - resolved_api_base = api_base or os.getenv(self.api_base_env) - if not resolved_api_key or not resolved_api_base: - return candidates - - try: - query_embedding = await self._embed_texts( - api_key=resolved_api_key, - api_base=resolved_api_base, - texts=[query], - model=model or self.model, - ) - candidate_texts = [self._candidate_text(item) for item in candidates] - candidate_embeddings = await self._embed_texts( - api_key=resolved_api_key, - api_base=resolved_api_base, - texts=candidate_texts, - model=model or self.model, - ) - except Exception: - return candidates - - if not query_embedding or not query_embedding[0] or len(candidate_embeddings) != len(candidates): - return candidates - - query_vector = query_embedding[0] - scored: list[tuple[float, dict[str, str]]] = [] - for candidate, vector in zip(candidates, candidate_embeddings, strict=False): - if not vector: - continue - scored.append((self._cosine_similarity(query_vector, vector), candidate)) - - scored.sort(key=lambda item: item[0], reverse=True) - return [item[1] for item in scored[:top_k]] - - async def _embed_texts( - self, - *, - api_key: str, - api_base: str, - texts: list[str], - model: str, - ) -> list[list[float]]: - """调用 OpenAI-compatible embeddings 接口。 - - 当前对齐的是你们实际在用的网关配置： - - `POST {api_base}/embeddings` - - `model=text-embedding-v4` - - `encoding_format=float` - """ - - all_vectors: list[list[float]] = [] - endpoint = self._normalize_embeddings_endpoint(api_base) - for start in range(0, len(texts), 10): - batch = texts[start:start + 10] - payload = await self._post_embeddings( - endpoint=endpoint, - api_key=api_key, - model=model, - texts=batch, - ) - embeddings = payload.get("data") or [] - embeddings = sorted(embeddings, key=lambda item: item.get("index", 0)) - all_vectors.extend([list(item.get("embedding") or []) for item in embeddings]) - return all_vectors - - async def _post_embeddings( - self, - *, - endpoint: str, - api_key: str, - model: str, - texts: list[str], - ) -> dict[str, Any]: - return await asyncio.to_thread( - self._post_embeddings_sync, - endpoint=endpoint, - api_key=api_key, - model=model, - texts=texts, - ) - - def _post_embeddings_sync( - self, - *, - endpoint: str, - api_key: str, - model: str, - texts: list[str], - ) -> dict[str, Any]: - body = json.dumps( - { - "model": model, - "input": texts if len(texts) > 1 else texts[0], - "encoding_format": "float", - } - ).encode("utf-8") - req = request.Request( - endpoint, - data=body, - headers={ - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - }, - method="POST", - ) - with request.urlopen(req, timeout=self.timeout_seconds) as response: - return json.loads(response.read().decode("utf-8")) - - @staticmethod - def _candidate_text(candidate: dict[str, str]) -> str: - name = (candidate.get("name") or "").strip() - description = (candidate.get("description") or "").strip() - return f"{name}\n{description}".strip() - - @staticmethod - def _normalize_embeddings_endpoint(api_base: str) -> str: - base = api_base.rstrip("/") - if base.endswith("/embeddings"): - return base - if base.endswith("/v1"): - return f"{base}/embeddings" - return f"{base}/v1/embeddings" - - @staticmethod - def _cosine_similarity(left: list[float], right: list[float]) -> float: - if not left or not right or len(left) != len(right): - return -1.0 - dot = sum(a * b for a, b in zip(left, right, strict=False)) - left_norm = math.sqrt(sum(a * a for a in left)) - right_norm = math.sqrt(sum(b * b for b in right)) - if left_norm == 0 or right_norm == 0: - return -1.0 - return dot / (left_norm * right_norm) diff --git a/app-instance/backend/beaver/skills/assembler/task_assembler.py b/app-instance/backend/beaver/skills/assembler/task_assembler.py index 876f593..c84f47d 100644 --- a/app-instance/backend/beaver/skills/assembler/task_assembler.py +++ b/app-instance/backend/beaver/skills/assembler/task_assembler.py @@ -63,6 +63,11 @@ class SkillAssembler: api_key=embedding_runtime.api_key if embedding_runtime is not None else None, api_base=embedding_runtime.api_base if embedding_runtime is not None else None, model=embedding_runtime.model if embedding_runtime is not None else None, + extra_headers=embedding_runtime.extra_headers if embedding_runtime is not None else None, + timeout_seconds=( + embedding_runtime.request_timeout_seconds if embedding_runtime is not None else None + ), + fallback_top_k=None, ) if not candidates: return SkillAssemblyResult() diff --git a/app-instance/backend/beaver/skills/catalog/loader.py b/app-instance/backend/beaver/skills/catalog/loader.py index 82516d8..8961141 100644 --- a/app-instance/backend/beaver/skills/catalog/loader.py +++ b/app-instance/backend/beaver/skills/catalog/loader.py @@ -18,6 +18,7 @@ from __future__ import annotations from dataclasses import dataclass +import json from pathlib import Path from typing import Any @@ -111,6 +112,32 @@ class SkillsLoader: metadata, _ = parse_frontmatter(content) return metadata + def get_skill_tool_hints(self, name: str) -> list[str]: + """读取 skill 显式声明的推荐工具。 + + 第一版只信任显式 metadata，不从正文里猜： + - `tools: read_file, search_files` + - `tools: ["read_file", "search_files"]` + - YAML-like list: + tools: + - read_file + - search_files + - 兼容 metadata JSON blob 里的 `tools` + """ + + frontmatter = self.get_skill_metadata(name) or {} + meta_blob = parse_skill_metadata_blob(frontmatter.get("metadata", "")) + names = [ + *self._coerce_tool_names(frontmatter.get("tools")), + *self._coerce_tool_names(meta_blob.get("tools")), + *self._coerce_tool_names(meta_blob.get("required_tools")), + ] + result: list[str] = [] + for item in names: + if item and item not in result: + result.append(item) + return result + def load_skills_for_context(self, skill_names: list[str]) -> str: """加载指定 skills 的正文，并整理成上下文块。""" @@ -253,6 +280,26 @@ class SkillsLoader: result.append(record.name) return result + @staticmethod + def _coerce_tool_names(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + raw = value.strip() + if not raw: + return [] + if raw.startswith("["): + try: + parsed = json.loads(raw) + except Exception: + parsed = None + if isinstance(parsed, list): + return [str(item).strip() for item in parsed if str(item).strip()] + return [item.strip() for item in raw.split(",") if item.strip()] + if isinstance(value, (list, tuple, set)): + return [str(item).strip() for item in value if str(item).strip()] + return [] + def _find_record(self, name: str) -> SkillRecord | None: for record in self.list_skills(filter_unavailable=False): if record.name == name: diff --git a/app-instance/backend/beaver/skills/catalog/utils.py b/app-instance/backend/beaver/skills/catalog/utils.py index 14e8fcc..8d8ded3 100644 --- a/app-instance/backend/beaver/skills/catalog/utils.py +++ b/app-instance/backend/beaver/skills/catalog/utils.py @@ -20,7 +20,7 @@ import shutil from typing import Any -def parse_frontmatter(content: str) -> tuple[dict[str, str], str]: +def parse_frontmatter(content: str) -> tuple[dict[str, Any], str]: """解析 Markdown 文件顶部的极简 frontmatter。 当前先只支持最常见的： @@ -43,12 +43,36 @@ def parse_frontmatter(content: str) -> tuple[dict[str, str], str]: if match is None: return {}, content - metadata: dict[str, str] = {} - for line in match.group(1).splitlines(): + metadata: dict[str, Any] = {} + lines = match.group(1).splitlines() + index = 0 + while index < len(lines): + line = lines[index] if ":" not in line: + index += 1 continue key, value = line.split(":", 1) - metadata[key.strip()] = value.strip().strip('"\'') + key = key.strip() + value = value.strip() + if not value: + items: list[str] = [] + lookahead = index + 1 + while lookahead < len(lines): + candidate = lines[lookahead] + stripped = candidate.strip() + if not stripped: + lookahead += 1 + continue + if not stripped.startswith("- "): + break + items.append(stripped[2:].strip().strip('"\'')) + lookahead += 1 + if items: + metadata[key] = items + index = lookahead + continue + metadata[key] = value.strip('"\'') + index += 1 body = content[match.end():].strip() return metadata, body diff --git a/app-instance/backend/beaver/tools/__init__.py b/app-instance/backend/beaver/tools/__init__.py index df94b79..67f9955 100644 --- a/app-instance/backend/beaver/tools/__init__.py +++ b/app-instance/backend/beaver/tools/__init__.py @@ -1,6 +1,7 @@ """Tool system for Beaver.""" from .base import BaseTool, ObjectBackedTool, ToolContext, ToolResult, ToolSpec +from .assembler import ToolAssembler from .registry import ToolRegistry from .runtime import ToolExecutor @@ -8,6 +9,7 @@ __all__ = [ "BaseTool", "ObjectBackedTool", "ToolContext", + "ToolAssembler", "ToolExecutor", "ToolRegistry", "ToolResult", diff --git a/app-instance/backend/beaver/tools/assembler/__init__.py b/app-instance/backend/beaver/tools/assembler/__init__.py new file mode 100644 index 0000000..e428af7 --- /dev/null +++ b/app-instance/backend/beaver/tools/assembler/__init__.py @@ -0,0 +1,5 @@ +"""Tool selection for a single Beaver run.""" + +from .task_assembler import ToolAssembler + +__all__ = ["ToolAssembler"] diff --git a/app-instance/backend/beaver/tools/assembler/task_assembler.py b/app-instance/backend/beaver/tools/assembler/task_assembler.py new file mode 100644 index 0000000..ec18a94 --- /dev/null +++ b/app-instance/backend/beaver/tools/assembler/task_assembler.py @@ -0,0 +1,106 @@ +"""Task-driven tool assembler. + +这层和 SkillAssembler 的位置类似：它不执行工具，只决定本轮 run 应该把哪些 +tool schema 暴露给模型。 +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from beaver.engine.context import SkillContext +from beaver.foundation.embedding import EmbeddingRetriever +from beaver.tools.base import ToolSpec +from beaver.tools.registry import ToolRegistry + +if TYPE_CHECKING: + from beaver.engine.providers.runtime import ProviderRuntime + from beaver.skills.catalog.loader import SkillsLoader + + +class ToolAssembler: + """Use skill hints and embedding retrieval to select run-scoped tools.""" + + def __init__( + self, + *, + retriever: EmbeddingRetriever | None = None, + always_tool_names: Sequence[str] | None = None, + ) -> None: + self.retriever = retriever or EmbeddingRetriever() + self.always_tool_names = tuple(always_tool_names or ("memory", "session_search", "skill_view")) + + async def assemble( + self, + *, + task_description: str, + registry: ToolRegistry, + skills_loader: SkillsLoader | None = None, + activated_skills: Sequence[SkillContext] | None = None, + embedding_runtime: ProviderRuntime | None = None, + top_k: int = 10, + ) -> list[ToolSpec]: + """Return selected tool specs for the current run. + + Selection order is intentionally deterministic: + 1. always tools from config/spec + 2. tools explicitly declared by activated skills + 3. embedding top-k tools for the task + """ + + selected: list[ToolSpec] = [] + selected_names: set[str] = set() + + def add_specs(specs: Sequence[ToolSpec]) -> None: + for spec in specs: + if spec.name in selected_names: + continue + selected.append(spec) + selected_names.add(spec.name) + + add_specs(registry.list_always_specs()) + add_specs(registry.get_specs(self.always_tool_names)) + + skill_tool_names = self._collect_skill_tool_names( + skills_loader=skills_loader, + activated_skills=activated_skills or (), + ) + add_specs(registry.get_specs(skill_tool_names)) + + candidates = [ + spec.to_embedding_candidate() + for spec in registry.list_specs() + if spec.name not in selected_names + ] + retrieved = await self.retriever.retrieve( + query=task_description, + candidates=candidates, + top_k=top_k, + api_key=embedding_runtime.api_key if embedding_runtime is not None else None, + api_base=embedding_runtime.api_base if embedding_runtime is not None else None, + model=embedding_runtime.model if embedding_runtime is not None else None, + extra_headers=embedding_runtime.extra_headers if embedding_runtime is not None else None, + timeout_seconds=( + embedding_runtime.request_timeout_seconds if embedding_runtime is not None else None + ), + fallback_top_k=top_k, + ) + add_specs(registry.get_specs([item["name"] for item in retrieved])) + return selected + + @staticmethod + def _collect_skill_tool_names( + *, + skills_loader: SkillsLoader | None, + activated_skills: Sequence[SkillContext], + ) -> list[str]: + if skills_loader is None or not activated_skills: + return [] + + result: list[str] = [] + for skill in activated_skills: + for name in skills_loader.get_skill_tool_hints(skill.name): + if name not in result: + result.append(name) + return result diff --git a/app-instance/backend/beaver/tools/base.py b/app-instance/backend/beaver/tools/base.py index a19c990..a107f19 100644 --- a/app-instance/backend/beaver/tools/base.py +++ b/app-instance/backend/beaver/tools/base.py @@ -29,13 +29,30 @@ class ToolSpec: """单个工具对外暴露的描述信息。 这份信息主要服务两个场景： - 1. 导出给 provider 的 function schema - 2. 在 registry 中做列出、查找、调试 + 1. 以 MCP-style descriptor 作为统一事实来源 + 2. 导出给 provider 的 function schema + 3. 在 registry 中做列出、查找、调试与 embedding 召回 """ name: str description: str input_schema: dict[str, Any] + toolset: str = "core" + always_available: bool = False + + def to_mcp_descriptor(self) -> dict[str, Any]: + """导出 MCP ListTools 风格的工具描述。 + + MCP 的基础字段是 `name`、`description`、`inputSchema`。 + Beaver 内部额外的 toolset/always_available 不塞进这个对象， + 避免未来对接真实 MCP server 时出现格式偏差。 + """ + + return { + "name": self.name, + "description": self.description, + "inputSchema": self.input_schema, + } def to_provider_schema(self) -> dict[str, Any]: """导出为 OpenAI-compatible function tool schema。""" @@ -49,6 +66,15 @@ class ToolSpec: }, } + def to_embedding_candidate(self) -> dict[str, str]: + """导出给语义召回使用的轻量文本候选。""" + + return { + "name": self.name, + "description": self.description, + "input_schema": json.dumps(self.input_schema, ensure_ascii=False, sort_keys=True), + } + @dataclass(slots=True) class ToolContext: @@ -113,6 +139,8 @@ class ObjectBackedTool(BaseTool): name=str(getattr(backend, "name")), description=str(getattr(backend, "description", "")), input_schema=dict(getattr(backend, "parameters", {"type": "object", "properties": {}})), + toolset=str(getattr(backend, "toolset", "core")), + always_available=bool(getattr(backend, "always_available", False)), ) @property @@ -150,6 +178,8 @@ class ObjectBackedTool(BaseTool): if "current_session_id" not in arguments and hasattr(self.backend, "current_session_id"): arguments["current_session_id"] = context.session_id + if "workspace" not in arguments and hasattr(self.backend, "workspace"): + arguments["workspace"] = context.workspace @staticmethod def _normalize_output(content: Any) -> dict[str, Any]: diff --git a/app-instance/backend/beaver/tools/builtins/__init__.py b/app-instance/backend/beaver/tools/builtins/__init__.py index 4bfe1b1..d463d46 100644 --- a/app-instance/backend/beaver/tools/builtins/__init__.py +++ b/app-instance/backend/beaver/tools/builtins/__init__.py @@ -1,13 +1,17 @@ """Built-in Beaver tools.""" from .echo import EchoTool, echo_tool +from .filesystem import ListDirectoryTool, ReadFileTool, SearchFilesTool from .memory import MemoryTool, memory_tool from .skill_view import SkillViewTool, skill_view from .session_search import SessionSearchTool, session_search __all__ = [ "EchoTool", + "ListDirectoryTool", "MemoryTool", + "ReadFileTool", + "SearchFilesTool", "SkillViewTool", "SessionSearchTool", "echo_tool", diff --git a/app-instance/backend/beaver/tools/builtins/echo.py b/app-instance/backend/beaver/tools/builtins/echo.py index bea1358..0181389 100644 --- a/app-instance/backend/beaver/tools/builtins/echo.py +++ b/app-instance/backend/beaver/tools/builtins/echo.py @@ -34,6 +34,8 @@ class EchoTool: name: str = "echo" description: str = ECHO_TOOL_DESCRIPTION + toolset: str = "debug" + always_available: bool = False parameters: dict[str, Any] = field(default_factory=lambda: dict(ECHO_TOOL_PARAMETERS)) async def execute(self, **kwargs: Any) -> str: diff --git a/app-instance/backend/beaver/tools/builtins/filesystem.py b/app-instance/backend/beaver/tools/builtins/filesystem.py new file mode 100644 index 0000000..3f0f5bd --- /dev/null +++ b/app-instance/backend/beaver/tools/builtins/filesystem.py @@ -0,0 +1,442 @@ +"""Workspace-scoped read-only filesystem tools. + +这些工具是 Beaver 第一批真实本地工具，只做只读能力： +- list_directory +- read_file +- search_files + +安全边界先保持非常明确：所有用户传入路径都必须解析到当前 +`ToolContext.workspace` 内部。即使 workspace 里有指向外部的符号链接， +读取时也会因为真实路径越界而被拒绝。 +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import json +from pathlib import Path +from typing import Any, Iterable + + +MAX_LIST_ENTRIES = 1_000 +MAX_READ_LINES = 1_000 +MAX_READ_CHARS = 120_000 +MAX_SEARCH_RESULTS = 200 +MAX_SEARCH_FILE_BYTES = 2_000_000 +MAX_SEARCH_FILES = 5_000 +SKIP_DIR_NAMES = { + ".git", + ".hg", + ".svn", + ".venv", + "venv", + "__pycache__", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + "node_modules", + "dist", + "build", +} + + +LIST_DIRECTORY_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "path": { + "type": "string", + "default": ".", + "description": "Directory path relative to the current workspace. Absolute paths are allowed only if they stay inside the workspace.", + }, + "recursive": { + "type": "boolean", + "default": False, + "description": "Whether to recursively list child entries. Symlink directories are not followed.", + }, + "max_entries": { + "type": "integer", + "default": 200, + "minimum": 1, + "maximum": MAX_LIST_ENTRIES, + "description": "Maximum number of entries to return.", + }, + }, + "required": [], +} + +READ_FILE_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path relative to the current workspace. Absolute paths are allowed only if they stay inside the workspace.", + }, + "start_line": { + "type": "integer", + "default": 1, + "minimum": 1, + "description": "1-based line number to start reading from.", + }, + "max_lines": { + "type": "integer", + "default": 200, + "minimum": 1, + "maximum": MAX_READ_LINES, + "description": "Maximum number of lines to read.", + }, + }, + "required": ["path"], +} + +SEARCH_FILES_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Plain text query to search in file paths and UTF-8 text files.", + }, + "path": { + "type": "string", + "default": ".", + "description": "Directory or file path relative to the current workspace.", + }, + "max_results": { + "type": "integer", + "default": 50, + "minimum": 1, + "maximum": MAX_SEARCH_RESULTS, + "description": "Maximum number of matches to return.", + }, + "case_sensitive": { + "type": "boolean", + "default": False, + "description": "Whether search should be case-sensitive.", + }, + }, + "required": ["query"], +} + + +class WorkspacePathError(ValueError): + """Raised when a requested path escapes the configured workspace.""" + + +def _json_result(success: bool, **payload: Any) -> str: + return json.dumps({"success": success, **payload}, ensure_ascii=False, indent=2) + + +def _clamp_int(value: Any, *, default: int, minimum: int, maximum: int) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + parsed = default + return max(minimum, min(parsed, maximum)) + + +def _workspace_root(workspace: str | None) -> Path: + if not workspace: + raise WorkspacePathError("workspace is not configured for filesystem tools") + root = Path(workspace).expanduser().resolve(strict=True) + if not root.is_dir(): + raise WorkspacePathError(f"workspace is not a directory: {root}") + return root + + +def _resolve_existing_path(workspace: str | None, user_path: str | None) -> tuple[Path, Path]: + """Resolve a user path and ensure the real target stays inside workspace.""" + + root = _workspace_root(workspace) + raw_path = Path(user_path or ".").expanduser() + candidate = raw_path if raw_path.is_absolute() else root / raw_path + resolved = candidate.resolve(strict=True) + try: + resolved.relative_to(root) + except ValueError as exc: + raise WorkspacePathError( + f"path escapes workspace: {user_path or '.'}" + ) from exc + return root, resolved + + +def _relative_path(root: Path, path: Path) -> str: + try: + return str(path.relative_to(root)) or "." + except ValueError: + return str(path) + + +def _entry_type(path: Path) -> str: + if path.is_symlink(): + return "symlink" + if path.is_dir(): + return "directory" + if path.is_file(): + return "file" + return "other" + + +def _entry_payload(root: Path, path: Path) -> dict[str, Any]: + try: + stat = path.lstat() if path.is_symlink() else path.stat() + size = stat.st_size + except OSError: + size = None + return { + "name": path.name, + "path": _relative_path(root, path), + "type": _entry_type(path), + "size": size, + } + + +def _iter_directory(root: Path, directory: Path, *, recursive: bool) -> Iterable[Path]: + def sort_key(item: Path) -> tuple[bool, str]: + is_real_directory = not item.is_symlink() and item.is_dir() + return (not is_real_directory, item.name.lower()) + + entries = sorted(directory.iterdir(), key=sort_key) + for entry in entries: + yield entry + if not recursive or entry.is_symlink() or not entry.is_dir(): + continue + yield from _iter_directory(root, entry, recursive=True) + + +def _looks_binary(path: Path) -> bool: + try: + with path.open("rb") as handle: + sample = handle.read(4096) + except OSError: + return True + return b"\0" in sample + + +def _read_text_file(path: Path) -> str: + if _looks_binary(path): + raise ValueError("binary files cannot be read by read_file/search_files") + return path.read_text(encoding="utf-8") + + +def _iter_search_files(root: Path, start: Path) -> Iterable[Path]: + if start.is_file(): + yield start + return + + stack = [start] + visited = 0 + while stack and visited < MAX_SEARCH_FILES: + current = stack.pop() + try: + children = sorted(current.iterdir(), key=lambda item: item.name.lower()) + except OSError: + continue + + for child in children: + if child.is_symlink(): + continue + if child.is_dir(): + if child.name in SKIP_DIR_NAMES: + continue + stack.append(child) + continue + if child.is_file(): + visited += 1 + yield child + if visited >= MAX_SEARCH_FILES: + break + + +@dataclass(slots=True) +class ListDirectoryTool: + """List files and directories inside the current workspace.""" + + name: str = "list_directory" + description: str = ( + "List files and directories inside the current workspace. " + "Use this before reading files when you need to inspect project structure. " + "This tool never follows paths outside the workspace." + ) + toolset: str = "filesystem" + always_available: bool = True + workspace: str | None = None + parameters: dict[str, Any] = field(default_factory=lambda: dict(LIST_DIRECTORY_PARAMETERS)) + + async def execute( + self, + *, + path: str = ".", + recursive: bool = False, + max_entries: int = 200, + workspace: str | None = None, + ) -> str: + try: + root, resolved = _resolve_existing_path(workspace, path) + if not resolved.is_dir(): + return _json_result(False, error="not_a_directory", path=path) + + limit = _clamp_int(max_entries, default=200, minimum=1, maximum=MAX_LIST_ENTRIES) + entries: list[dict[str, Any]] = [] + truncated = False + for entry in _iter_directory(root, resolved, recursive=bool(recursive)): + entries.append(_entry_payload(root, entry)) + if len(entries) >= limit: + truncated = True + break + + return _json_result( + True, + path=_relative_path(root, resolved), + recursive=bool(recursive), + entries=entries, + truncated=truncated, + ) + except (OSError, WorkspacePathError, ValueError) as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class ReadFileTool: + """Read a UTF-8 text file inside the current workspace.""" + + name: str = "read_file" + description: str = ( + "Read a UTF-8 text file inside the current workspace with line limits. " + "Use this to inspect source code, docs, config, or logs. " + "This tool rejects binary files and paths outside the workspace." + ) + toolset: str = "filesystem" + always_available: bool = True + workspace: str | None = None + parameters: dict[str, Any] = field(default_factory=lambda: dict(READ_FILE_PARAMETERS)) + + async def execute( + self, + *, + path: str, + start_line: int = 1, + max_lines: int = 200, + workspace: str | None = None, + ) -> str: + try: + root, resolved = _resolve_existing_path(workspace, path) + if not resolved.is_file(): + return _json_result(False, error="not_a_file", path=path) + + start = _clamp_int(start_line, default=1, minimum=1, maximum=10_000_000) + limit = _clamp_int(max_lines, default=200, minimum=1, maximum=MAX_READ_LINES) + content = _read_text_file(resolved) + lines = content.splitlines() + selected = lines[start - 1 : start - 1 + limit] + selected_text = "\n".join(selected) + char_truncated = False + if len(selected_text) > MAX_READ_CHARS: + selected_text = selected_text[:MAX_READ_CHARS] + char_truncated = True + + end_line = start + len(selected) - 1 if selected else start - 1 + return _json_result( + True, + path=_relative_path(root, resolved), + start_line=start, + end_line=end_line, + total_lines=len(lines), + truncated=end_line < len(lines) or char_truncated, + content=selected_text, + ) + except UnicodeDecodeError: + return _json_result(False, error="file is not valid UTF-8 text", path=path) + except (OSError, WorkspacePathError, ValueError) as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class SearchFilesTool: + """Search filenames and UTF-8 text file contents inside the workspace.""" + + name: str = "search_files" + description: str = ( + "Search file paths and UTF-8 text file contents inside the current workspace. " + "Use this to find relevant source files, docs, config keys, or log lines. " + "This tool skips large/binary files and never searches outside the workspace." + ) + toolset: str = "filesystem" + always_available: bool = True + workspace: str | None = None + parameters: dict[str, Any] = field(default_factory=lambda: dict(SEARCH_FILES_PARAMETERS)) + + async def execute( + self, + *, + query: str, + path: str = ".", + max_results: int = 50, + case_sensitive: bool = False, + workspace: str | None = None, + ) -> str: + try: + if not isinstance(query, str) or not query.strip(): + return _json_result(False, error="query must be a non-empty string") + root, resolved = _resolve_existing_path(workspace, path) + if not resolved.is_dir() and not resolved.is_file(): + return _json_result(False, error="path must be a file or directory", path=path) + + limit = _clamp_int(max_results, default=50, minimum=1, maximum=MAX_SEARCH_RESULTS) + needle = query if case_sensitive else query.lower() + results: list[dict[str, Any]] = [] + searched_files = 0 + skipped_files = 0 + + for file_path in _iter_search_files(root, resolved): + relative = _relative_path(root, file_path) + haystack_path = relative if case_sensitive else relative.lower() + if needle in haystack_path: + results.append( + { + "path": relative, + "line": None, + "match_type": "path", + "preview": relative, + } + ) + if len(results) >= limit: + break + + try: + if file_path.stat().st_size > MAX_SEARCH_FILE_BYTES or _looks_binary(file_path): + skipped_files += 1 + continue + text = file_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + skipped_files += 1 + continue + + searched_files += 1 + lines = text.splitlines() + for index, line in enumerate(lines, start=1): + haystack_line = line if case_sensitive else line.lower() + if needle not in haystack_line: + continue + results.append( + { + "path": relative, + "line": index, + "match_type": "content", + "preview": line[:500], + } + ) + if len(results) >= limit: + break + if len(results) >= limit: + break + + return _json_result( + True, + query=query, + path=_relative_path(root, resolved), + results=results, + truncated=len(results) >= limit, + searched_files=searched_files, + skipped_files=skipped_files, + ) + except (OSError, WorkspacePathError, ValueError) as exc: + return _json_result(False, error=str(exc), path=path) diff --git a/app-instance/backend/beaver/tools/builtins/memory.py b/app-instance/backend/beaver/tools/builtins/memory.py index f510070..9ed2c06 100644 --- a/app-instance/backend/beaver/tools/builtins/memory.py +++ b/app-instance/backend/beaver/tools/builtins/memory.py @@ -123,6 +123,8 @@ class MemoryTool: store: MemoryStore name: str = "memory" description: str = MEMORY_TOOL_DESCRIPTION + toolset: str = "memory" + always_available: bool = True parameters: dict[str, Any] = field(default_factory=lambda: dict(MEMORY_TOOL_PARAMETERS)) async def execute(self, **kwargs: Any) -> str: diff --git a/app-instance/backend/beaver/tools/builtins/session_search.py b/app-instance/backend/beaver/tools/builtins/session_search.py index 7fec9cb..3125cbe 100644 --- a/app-instance/backend/beaver/tools/builtins/session_search.py +++ b/app-instance/backend/beaver/tools/builtins/session_search.py @@ -406,6 +406,8 @@ class SessionSearchTool: summarizer: SessionSummarizer | None = None name: str = "session_search" description: str = SESSION_SEARCH_TOOL_DESCRIPTION + toolset: str = "session" + always_available: bool = True parameters: dict[str, Any] = field(default_factory=lambda: dict(SESSION_SEARCH_TOOL_PARAMETERS)) async def execute(self, **kwargs: Any) -> str: diff --git a/app-instance/backend/beaver/tools/builtins/skill_view.py b/app-instance/backend/beaver/tools/builtins/skill_view.py index cc650eb..83a382b 100644 --- a/app-instance/backend/beaver/tools/builtins/skill_view.py +++ b/app-instance/backend/beaver/tools/builtins/skill_view.py @@ -76,6 +76,8 @@ class SkillViewTool: loader: SkillsLoader name: str = "skill_view" description: str = SKILL_VIEW_TOOL_DESCRIPTION + toolset: str = "skills" + always_available: bool = True parameters: dict[str, Any] = field(default_factory=lambda: dict(SKILL_VIEW_TOOL_PARAMETERS)) async def execute(self, **kwargs: Any) -> str: diff --git a/app-instance/backend/beaver/tools/registry/tool_registry.py b/app-instance/backend/beaver/tools/registry/tool_registry.py index 5799c65..9582e82 100644 --- a/app-instance/backend/beaver/tools/registry/tool_registry.py +++ b/app-instance/backend/beaver/tools/registry/tool_registry.py @@ -11,6 +11,7 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Iterable from beaver.tools.base import BaseTool, ToolSpec @@ -49,7 +50,30 @@ class ToolRegistry: def list_specs(self) -> list[ToolSpec]: return [tool.spec for tool in self._tools.values()] + def list_always_specs(self) -> list[ToolSpec]: + """列出每轮 run 都应该暴露给模型的基础工具。""" + + return [spec for spec in self.list_specs() if spec.always_available] + + def get_specs(self, names: Sequence[str]) -> list[ToolSpec]: + """按名称顺序返回已注册工具 spec，忽略未知工具。""" + + specs: list[ToolSpec] = [] + seen: set[str] = set() + for name in names: + tool = self.get(name) + if tool is None or name in seen: + continue + specs.append(tool.spec) + seen.add(name) + return specs + def export_provider_schemas(self) -> list[dict]: """导出给 provider 的函数工具 schema 列表。""" return [spec.to_provider_schema() for spec in self.list_specs()] + + def export_selected_provider_schemas(self, specs: Sequence[ToolSpec]) -> list[dict]: + """导出一组已选择工具的 provider schema。""" + + return [spec.to_provider_schema() for spec in specs] diff --git a/app-instance/backend/beaver/tools/runtime/executor.py b/app-instance/backend/beaver/tools/runtime/executor.py index 57a3366..6241df1 100644 --- a/app-instance/backend/beaver/tools/runtime/executor.py +++ b/app-instance/backend/beaver/tools/runtime/executor.py @@ -12,12 +12,14 @@ from __future__ import annotations import json -from typing import Any +from typing import TYPE_CHECKING, Any -from beaver.engine.providers.base import ToolCallRequest from beaver.tools.base import ToolContext, ToolResult from beaver.tools.registry.tool_registry import ToolRegistry +if TYPE_CHECKING: + from beaver.engine.providers.base import ToolCallRequest + class ToolExecutor: """统一执行单个 tool call。""" @@ -80,16 +82,17 @@ class ToolExecutor: @staticmethod def _normalize_tool_call(tool_call: ToolCallRequest | dict[str, Any]) -> tuple[str, dict[str, Any]]: - if isinstance(tool_call, ToolCallRequest): - return tool_call.name, dict(tool_call.arguments) - - function = tool_call.get("function") - if isinstance(function, dict): - name = function.get("name") - arguments = function.get("arguments", {}) + if not isinstance(tool_call, dict): + name = getattr(tool_call, "name", None) + arguments = getattr(tool_call, "arguments", {}) else: - name = tool_call.get("name") - arguments = tool_call.get("arguments", {}) + function = tool_call.get("function") + if isinstance(function, dict): + name = function.get("name") + arguments = function.get("arguments", {}) + else: + name = tool_call.get("name") + arguments = tool_call.get("arguments", {}) if not name: raise ValueError("Tool call is missing a tool name") @@ -104,8 +107,8 @@ class ToolExecutor: @staticmethod def _extract_tool_name(tool_call: ToolCallRequest | dict[str, Any]) -> str: - if isinstance(tool_call, ToolCallRequest): - return str(tool_call.name or "unknown") + if not isinstance(tool_call, dict): + return str(getattr(tool_call, "name", None) or "unknown") function = tool_call.get("function") if isinstance(function, dict) and function.get("name"): return str(function["name"]) diff --git a/app-instance/backend/flow.md b/app-instance/backend/flow.md index 28a30ea..5be8496 100644 --- a/app-instance/backend/flow.md +++ b/app-instance/backend/flow.md @@ -73,27 +73,49 @@ 1. `4.1 session` 2. `4.2 provider` 3. `4.3 context` -4. `4.4 tools` +4. `4.4 tools framework + 最小内建工具` 5. `4.5 最小主链` 6. `5.1 memory 最小接入` 7. `5.2 skills 最小接入` 8. `6.1 session-first / event-source 第一阶段` +9. `6.2 runtime lifecycle 最小骨架` +10. `6.2.1 Web / Gateway 最小接主链` +11. app-instance Docker 镜像切到新 `beaver` 后端 更准确地说，当前 Beaver 已经有： 1. 一个可运行的 `AgentService -> AgentLoop` 主链 2. 一个外部化的 Session 子系统 -3. 一个可工作的 tool loop +3. 一个可工作的 tool loop 框架 4. Hermes 风格的 memory / skills 接入 5. LLM-driven 的 `SkillAssembler` +6. embedding-driven 的 `ToolAssembler` +7. MCP-style 本地工具描述 +8. skill frontmatter `tools` 会影响本轮工具选择 +9. `start()/submit_direct()/stop()/shutdown()/close()` 最小 lifecycle +10. FastAPI `/api/ping` + `/api/chat` +11. Gateway `MessageBus -> AgentService -> MessageBus` 最小桥接 +12. Docker app-instance 使用 `/root/.beaver/config.json` 和 `/root/.beaver/workspace` + +已经实测通过： + +1. Docker image build +2. container `/api/ping` +3. `/api/chat` 调用 `qwen-plus` +4. Session SQLite 事件写入 +5. 宿主机 `curl` 直连 app-instance 但还没有： -1. 更完整的 shutdown hooks -2. Web / Gateway 的 bus / channels / realtime 全量接入 -3. delegation / swarm / team runtime -4. 权限系统 -5. MCP 全量工具接回 runtime +1. shell / web 等高风险或外部访问工具 +2. 完整 tool permission gates +3. Web / Gateway 的 realtime streaming +4. bus retry / routing / persistence +5. delegation / swarm / team runtime +6. MCP 全量工具接回 runtime +7. checkpoint / rewind / fork / crash-resume +8. skill selector 的 embedding / LLM 选择细节还没有写入 Session event stream +9. 前端完整 auth / sessions / skills / files / ws 兼容新 Beaver API --- @@ -106,7 +128,7 @@ service = AgentService() await service.process_direct("你好") ``` -同时，第 6 阶段的最小运行循环已经有了： +上面是 direct/debug path。宿主层进入运行模式后，正式入口是： ```python service = AgentService() @@ -123,6 +145,36 @@ app = create_app() # FastAPI lifespan 内部托管 AgentService.start()/s await run_gateway() # Gateway 常驻进程托管 AgentService.start()/shutdown() ``` +模型与 provider 配置现在从 backend sandbox config 统一读取，而不是从前端或 channel +请求里传密钥。Docker 单实例部署时，配置路径优先级是： + +1. `BEAVER_CONFIG_PATH` +2. `NANOBOT_CONFIG_PATH` +3. `BEAVER_HOME/config.json` +4. `NANOBOT_HOME/config.json` +5. `/.beaver/config.json` + +当前 app-instance 会把每个用户实例自己的数据目录挂到 `/root/.beaver`，所以 +Beaver 会默认读取： + +```text +/root/.beaver/config.json +``` + +这份配置跟随单个 sandbox 容器/数据卷，不放在前端，也不放在宿主机全局目录。 +Web / Gateway / Channel 只传 `message/session_id/user_id` 等业务输入。 + +app-instance 镜像当前也已经切到新 Beaver 后端： + +```text +entrypoint.sh +├─ 启动 python -m uvicorn beaver.interfaces.web.app:create_app --factory +├─ 使用 /root/.beaver/config.json +└─ 使用 /root/.beaver/workspace +``` + +旧的 `nanobot web`、`backend/nanobot`、`backend/bridge`、vendored `swarms` 不再进入新镜像。 + 这套 lifecycle 当前明确是： 1. `start()` 进入一个 `AgentLoop` 实例的运行模式 @@ -165,15 +217,27 @@ await run_gateway() # Gateway 常驻进程托管 AgentService.start()/shut - `run_gateway()` 启动时： - 如果 gateway 自己创建 service，则 `await service.start()` - 持有最小 `MessageBus` - - 常驻消费 `bus.inbound` - - 调 `await service.submit_direct(...)` - - 把结果写回 `bus.outbound` + - 可选接收 `ChannelManager` / channel adapters + - `ChannelManager` 和 `channels` 参数二选一： + - 传 `ChannelManager`：外部提前配置好 channel + - 传 `channels`：gateway 内部创建 `ChannelManager` 并注册这些 channel + - inbound 流向： + - channel adapter 发布 `InboundMessage` + - `MessageBus.inbound` + - gateway bridge 常驻消费 + - `await service.handle_inbound_message(...)` + - outbound 流向： + - `AgentService` 内部完成 `InboundMessage -> OutboundMessage` 映射 + - gateway bridge 写回 `MessageBus.outbound` + - 如果启用了 `ChannelManager`，则分发给对应 channel adapter + - 未启用 `ChannelManager` 时，保留直接消费 `bus.outbound` 的最小测试能力 - 同时等待 `stop_event` - 退出时： - 先尝试 `await service.shutdown(timeout_seconds=5.0, force=True)` - 再等待 bridge 协程收尾；必要时取消 bridge - - 如果 gateway 自己接管 lifecycle 且 `start()` 失败： - - 会立即 `close()` 做 startup cleanup + - 再等待 outbound dispatch 协程收尾；必要时取消 dispatch + - 如果 gateway 自己接管 lifecycle 且 `start()` 失败： + - 会立即 `close()` 做 startup cleanup - 未处理完的 inbound： - 不再静默丢下 - 会被冲刷成结构化 outbound error @@ -188,6 +252,16 @@ await run_gateway() # Gateway 常驻进程托管 AgentService.start()/shut - `outbound` - 还没有 broker / topic routing / retry / persistence +4. `beaver/interfaces/channels/*` + - 已有最小 channel adapter 层： + - `ChannelAdapter` + - `ChannelManager` + - `MemoryChannelAdapter` + - 当前 channel 职责很窄： + - 把外部输入发布成 `InboundMessage` + - 接收并投递 `OutboundMessage` + - `MemoryChannelAdapter` 只用于本地测试和内嵌接入，不是正式消息 broker + 所以现在已经明确： 1. Web / Gateway 属于宿主层 @@ -202,9 +276,11 @@ await run_gateway() # Gateway 常驻进程托管 AgentService.start()/shut - 外部注入的 `AgentService`：默认不自动 start/shutdown，除非显式要求接管 5. gateway 已经从“只会常驻等待”推进到“最小消息桥接层” - external inbound message + - channel adapter - `MessageBus.inbound` - - `service.submit_direct(...)` + - `service.handle_inbound_message(...)` - `MessageBus.outbound` + - channel adapter outbound delivery ### 3.2 总体链路 @@ -216,6 +292,7 @@ AgentService -> Session -> Memory -> SkillAssembler + -> ToolAssembler -> ContextBuilder -> Provider -> ToolExecutor @@ -237,6 +314,7 @@ AgentService │ ├─ MemoryStore │ ├─ MemoryService │ ├─ ToolRegistry +│ ├─ ToolAssembler │ ├─ ToolExecutor │ ├─ SkillsLoader │ ├─ SkillAssembler @@ -269,6 +347,18 @@ AgentService │ ├─ 如果 activated_skills 非空： │ │ └─ sessions.append_message(event_type="skill_activation_snapshotted", hidden) │ │ +│ ├─ tool_assembler.assemble(task_description=task, activated_skills=..., ...) +│ │ ├─ always tools +│ │ │ ├─ memory +│ │ │ ├─ session_search +│ │ │ └─ skill_view +│ │ ├─ 读取 activated skill 的 frontmatter `tools` +│ │ ├─ 用 `text-embedding-v4` 对工具描述做相似度召回 +│ │ ├─ 返回本轮选中的 ToolSpec +│ │ └─ ToolSpec 同时可导出 MCP descriptor 与 provider schema +│ │ +│ ├─ sessions.append_message(event_type="tool_selection_snapshotted", hidden) +│ │ │ ├─ ContextBuilder.build_messages() │ │ ├─ system prompt 包含： │ │ │ ├─ base system prompt @@ -328,10 +418,11 @@ AgentService 2. `MemoryStore` 3. `MemoryService` 4. `ToolRegistry` -5. `ToolExecutor` -6. `SkillsLoader` -7. `SkillAssembler` -8. `ContextBuilder` +5. `ToolAssembler` +6. `ToolExecutor` +7. `SkillsLoader` +8. `SkillAssembler` +9. `ContextBuilder` ### 4.2 `AgentLoop` @@ -349,7 +440,7 @@ AgentService 1. 更复杂的 message bus mode 2. 多 worker / 并发调度 -3. 更完整的 runtime lifecycle +3. provider/client 级 async shutdown hooks 4. multi-agent orchestration ### 4.3 `Session` @@ -383,9 +474,10 @@ AgentService 1. `run_started` 2. `skill_activation_snapshotted` -3. `system_prompt_snapshotted` -4. `run_completed` -5. `run_failed` +3. `tool_selection_snapshotted` +4. `system_prompt_snapshotted` +5. `run_completed` +6. `run_failed` ### 4.4 `Memory` @@ -438,13 +530,57 @@ AgentService 2. `memory` 3. `skill_view` 4. `session_search` +5. `list_directory` +6. `read_file` +7. `search_files` 当前工具基础设施： 1. `ToolSpec` + - 以 MCP-style descriptor 作为本地统一描述 + - 可导出 `to_mcp_descriptor()` + - 可导出 OpenAI-compatible `to_provider_schema()` 2. `ObjectBackedTool` 3. `ToolRegistry` 4. `ToolExecutor` +5. `ToolAssembler` + +当前工具选择语义： + +1. 工具选择是 **run-scoped** +2. `memory` / `session_search` / `skill_view` / 只读 filesystem tools 是 always tools +3. activated skill 的 frontmatter 可声明： + +```yaml +--- +tools: + - terminal + - read_file +--- +``` + +4. `ToolAssembler` 会合并： + - always tools + - activated skill 显式声明的 tools + - task description embedding top10 tools +5. 当前只信任 frontmatter / metadata 里的显式 tools，不从 skill 正文里猜工具名 +6. 如果 skill 声明了未注册工具，当前会忽略，不阻断 run + +当前 filesystem tools 的边界： + +1. `list_directory` 只能列当前 `ToolContext.workspace` 内的目录 +2. `read_file` 只能读 workspace 内 UTF-8 文本文件 +3. `search_files` 只能搜索 workspace 内文件名和 UTF-8 文本内容 +4. 绝对路径如果解析后不在 workspace 内，会拒绝 +5. workspace 内指向外部的符号链接，读取 / 搜索时会拒绝 +6. 二进制文件会拒绝读取，并在搜索时跳过 + +当前还没有默认注册： + +1. shell / exec tools +2. web search / web fetch tools +3. MCP tools +4. spawn / team tools ### 4.7 `Providers` @@ -454,12 +590,16 @@ AgentService 2. runtime resolution 3. main provider 4. fallback provider +5. auxiliary provider +6. embedding runtime 配置线 当前状态： 1. fallback 已经是“每次调用都先 main，再 fallback” 2. auxiliary provider 已经可用于 skill 选择 -3. auxiliary provider 还没有进入主对话 tool loop +3. embedding runtime 当前用于 SkillAssembler 的候选召回 +4. embedding runtime 当前也用于 ToolAssembler 的工具召回 +5. auxiliary provider 还没有进入主对话 tool loop --- @@ -507,26 +647,81 @@ task description 1. activated skill messages 2. `skill_view` +### 5.5 `Tools` 采用 MCP-style 描述 + +当前本地工具不再只是一段 OpenAI function schema，而是先收敛成： + +```text +ToolSpec +├─ name +├─ description +├─ input_schema +├─ toolset +└─ always_available +``` + +其中 `name/description/input_schema` 可直接导出 MCP-style descriptor： + +```json +{ + "name": "memory", + "description": "...", + "inputSchema": {} +} +``` + +provider 需要的 OpenAI-compatible schema 由 `ToolSpec.to_provider_schema()` 转换出来。 + --- -## 6. 当前还没完成什么 +## 6. 对照施工指南，我们现在处于哪一步 -这部分是接下来继续施工的重点。 +这部分严格对齐 `施工指南.md` 的第 6 阶段编号，不再自行改号。 -### 6.1 运行时生命周期 +### 6.1 第一步：Session 升级为事件源模型 -已做第一步： +当前状态：**基本完成第一阶段目标，但还不是完整 event-source 系统。** + +已经具备： + +1. `messages` 表已经承担主事件流语义 +2. 每次 run 都有独立 `run_id` +3. `AgentLoop.process_direct()` 已按事件阶段写回 Session +4. 已有： + - `get_event_records(session_id)` + - `get_run_event_records(session_id, run_id)` + - `list_run_ids(session_id)` + - `get_visible_history(session_id)` +5. `session_search` 只检索可见 transcript，不把 hidden snapshots 当搜索候选 + +当前还没做： + +1. `checkpoint` +2. `rewind` +3. `fork session` +4. `crash-resume protocol` + +所以更准确地说： + +1. `6.1` 的“Session-first / event-source 第一阶段”已经落地 +2. 但更完整的 event-source 能力还没有做完 + +### 6.2 第二步：runtime 生命周期协议补齐 + +当前状态：**最小 lifecycle 骨架已经完成。** + +已完成： 1. `EngineLoadResult.close()` 2. `AgentLoop.close()` 3. `AgentService.close()` 4. `AgentService.shutdown()` - -已做第二步的最小版本： - -1. `AgentLoop.run()` -2. `AgentLoop.stop()` -3. `AgentLoop.submit_direct()` +5. `AgentLoop.run()` +6. `AgentLoop.stop()` +7. `AgentLoop.submit_direct()` +8. `AgentService.start()` +9. `AgentService.stop()` +10. `AgentService.submit_direct()` 还没做： @@ -534,66 +729,160 @@ task description 2. 更完整的 provider/client 资源释放协议 3. 多 worker / bus / 调度策略 -### 6.2 Web / Gateway 接主链 +### 6.2.1 Web / Gateway 现在如何接这套 lifecycle -现在主链已经能跑，但还没正式变成： +当前状态：**最小宿主层接入已经完成。** -1. Web 真正调用 `AgentService.process_direct()` -2. Gateway 真正调用 `AgentService.process_direct()` +已经完成： -### 6.3 Session 更完整的 event-source 能力 +1. Web 通过 FastAPI lifespan 托管 `AgentService.start()/shutdown()` +2. Web 请求只走 `AgentService.submit_direct()` +3. Gateway 已有最小 `MessageBus -> AgentService.handle_inbound_message() -> MessageBus` 桥接 +4. Gateway 已支持可选 `ChannelManager`，把 outbound 分发回 channel adapter -还没做： +当前 app-instance Docker 已完成： -1. checkpoint -2. rewind -3. fork session -4. crash-resume protocol +1. Dockerfile 只安装 `backend/beaver` +2. entrypoint 启动 `beaver.interfaces.web.app:create_app` +3. 每个实例挂载 `/root/.beaver` +4. 配置读取 `/root/.beaver/config.json` +5. workspace 使用 `/root/.beaver/workspace` +6. 宿主 `curl /api/chat` 已实测通过 -### 6.4 Multi-agent / swarms +这一小步还没做： -还没正式接回主链： +1. realtime streaming +2. retry / broker persistence +3. 外部真实 channel adapter 全量接入 -1. delegation -2. team runtime -3. swarms orchestration backend +### 6.3 第三步：回填 bus 模式 -但 lifecycle 关系已经先定下来了： +当前状态：**只完成了前置地基，还没有按施工指南真正收口。** + +已经具备的前置件： + +1. `MessageBus` +2. `InboundMessage` +3. `OutboundMessage` +4. `AgentService.handle_inbound_message()` +5. Gateway bridge 常驻消费 inbound 并写回 outbound +6. `AgentLoop.run()` 已有最小运行循环 + +但严格按 `施工指南.md` 来看，`6.3` 还没有正式完成，因为现在还缺： + +1. 把 bus mode 明确成 runtime 的正式运行形态之一 +2. 明确 `run()` 如何稳定消费 inbound message +3. 明确 bus mode 与 direct mode / queue mode 的职责边界 +4. 明确停机、取消、冲刷 pending inbound 时的统一语义 +5. 再决定后续是否需要更复杂的 worker / retry / routing + +也就是说： + +1. 现在不是“还没 bus” +2. 而是“已经把 bus 协议映射收口到 `AgentService`，但还没按施工指南把它扩成完整 bus runtime 模式” + +### 6.4 单 agent lifecycle 如何扩展到 team + +当前状态：**关系已经定死，但实现还没开始。** + +当前已经明确： 1. team 不会共享一个大 `AgentLoop` 跑所有成员 2. 每个 team member 都应有自己独立的 `AgentService / AgentLoop` 3. team coordinator 在上层调度多个 member 实例 4. 因此当前这套 `start()/submit_direct()/stop()/close()` 首先是 member-level lifecycle + +当前还没开始的部分： + +1. delegation 2. team runtime -3. swarms backend +3. swarms orchestration backend 4. group discussion / workflow orchestration -### 6.5 权限与治理 - -还没做： - -1. permission gates -2. tool policy -3. MCP 工具治理 - --- -## 7. 下一步从哪开始最合理 +## 7. 对照 `change.md`，哪些长期目标还没开始 -如果现在继续施工，最合理的顺序是： +`change.md` 讲的是总蓝图，不是当前施工编号。下面这些仍然是长期目标，还没有正式进入当前阶段实现： -1. 先把 `flow.md` 作为当前基线固定下来 -2. 再继续第 6 阶段： - - runtime lifecycle - - `boot / close / run / stop` -3. 然后再接： - - Web / Gateway -4. 最后才是： - - multi-agent / swarms +1. skills 生命周期系统 + - `SkillDraft` + - `SkillVersion` + - review / publish / rollback +2. Hermes-style learning loop + - 智能体定期整理 / 提示记忆 + - 复杂任务完成后可自主创建技能 + - 技能在使用过程中自我提升 + - FTS5 + LLM 摘要的跨会话回忆增强 + - Honcho 风格辩证用户建模 +3. swarms 作为正式 backend 接回平台 +4. delegation / subagent / team orchestration + +当前只完成了这些基础入口： + +1. curated memory CRUD +2. session_search +3. skill loader / skill_view +4. skill assembler +5. tool assembler + +### 7.1 权限与治理 + +还没做： + +1. 完整 permission gates +2. tool policy +3. MCP 工具治理 + +已完成的最小边界： + +1. 只读 filesystem tools 强制限制在 `ToolContext.workspace` +2. 路径解析使用真实路径，防止相对路径、绝对路径、符号链接逃逸 +3. 当前还没有 shell / write / network 工具，因此还没进入高风险授权阶段 + +### 7.2 前端兼容 + +当前只做了最小 chat response 兼容： + +1. 前端 `sendMessage()` 已兼容 Beaver 的 `output_text` + +还没做： + +1. `/api/auth/*` +2. `/api/sessions` +3. `/api/status` 完整页面数据 +4. `/api/skills` +5. `/api/files` +6. `/ws` +7. 浏览器端免登录或新 auth 接入策略 + +--- + +## 8. 下一步从哪开始最合理 + +如果严格按 `施工指南.md` 的施工顺序继续，下一步应是： + +1. 完成 `6.3 回填 bus 模式` + - 明确 bus mode 的正式运行语义 + - 让 `AgentLoop.run()` 与 `MessageBus` 的关系稳定收口 + - 把 inbound / outbound 结果结构定稳 +2. 然后再进入 `6.4` + - 先把 team lifecycle 关系写成更可实现的 coordinator 约束 +3. 再进入第 7 阶段 + - delegation + - local subagent +4. 再进入第 8 阶段 + - team / swarms backend + +如果按 `change.md` 的长期方向看，后面还要补： + +1. skills 生命周期 +2. Hermes-style learning loop +3. 更完整的 memory / governance / frontend 一句话总结： -**当前 Beaver 已经有一个可运行的单 agent runtime；接下来不是继续堆局部能力，而是把它升级成有完整生命周期的标准 harness。** +**当前 Beaver 已经完成到“单 agent runtime + memory/skills + lifecycle + Web/Gateway 最小接入”，按施工指南的编号，下一步应是 `6.3 回填 bus 模式`。** --- diff --git a/app-instance/backend/tests/unit/test_config_loader.py b/app-instance/backend/tests/unit/test_config_loader.py new file mode 100644 index 0000000..7840cde --- /dev/null +++ b/app-instance/backend/tests/unit/test_config_loader.py @@ -0,0 +1,107 @@ +import json + +from beaver.engine import AgentLoop, EngineLoader +from beaver.engine.providers import make_provider_bundle +from beaver.engine.providers.litellm import LiteLLMProvider +from beaver.foundation.config import load_config + + +def test_load_config_reads_current_instance_shape(tmp_path) -> None: + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps( + { + "agents": { + "defaults": { + "workspace": str(tmp_path / "workspace"), + "model": "qwen-plus", + } + }, + "providers": { + "openai": { + "apiKey": "sk-test", + "apiBase": "https://oai.example.com/v1", + "extraHeaders": {"X-Test": "1"}, + } + }, + "embeddingModel": "text-embedding-v4", + } + ), + encoding="utf-8", + ) + + config = load_config(config_path=config_path) + target = config.resolve_provider_target() + + assert config.default_model == "qwen-plus" + assert config.default_embedding_model == "text-embedding-v4" + assert target["provider_name"] == "openai" + assert target["model"] == "qwen-plus" + assert target["api_key"] == "sk-test" + assert target["api_base"] == "https://oai.example.com/v1" + assert target["extra_headers"] == {"X-Test": "1"} + + +def test_engine_loader_uses_config_workspace(tmp_path) -> None: + workspace = tmp_path / "workspace" + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps( + { + "agents": { + "defaults": { + "workspace": str(workspace), + "model": "qwen-plus", + } + }, + "providers": {"openai": {"apiKey": "sk-test", "apiBase": "https://oai.example.com/v1"}}, + } + ), + encoding="utf-8", + ) + + loader = EngineLoader(config_path=config_path) + assert loader.workspace == workspace + + +def test_agent_loop_config_drives_provider_bundle(tmp_path) -> None: + workspace = tmp_path / "workspace" + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps( + { + "agents": { + "defaults": { + "workspace": str(workspace), + "model": "qwen-plus", + } + }, + "providers": {"openai": {"apiKey": "sk-test", "apiBase": "https://oai.example.com/v1"}}, + } + ), + encoding="utf-8", + ) + + loop = AgentLoop(loader=EngineLoader(config_path=config_path)) + loaded = loop.boot() + target = loaded.config.resolve_provider_target() + + assert target["provider_name"] == "openai" + assert target["model"] == "qwen-plus" + assert target["api_key"] == "sk-test" + assert target["api_base"] == "https://oai.example.com/v1" + loop.close() + + +def test_openai_compatible_qwen_config_keeps_openai_provider() -> None: + bundle = make_provider_bundle( + model="qwen-plus", + provider_name="openai", + api_key="sk-test", + api_base="https://oai.example.com/v1", + ) + + assert bundle.main_runtime.provider_name == "openai" + assert bundle.main_runtime.api_base == "https://oai.example.com/v1" + assert isinstance(bundle.main_provider, LiteLLMProvider) + assert bundle.main_provider._resolve_model("qwen-plus") == "openai/qwen-plus" diff --git a/app-instance/backend/tests/unit/test_filesystem_tools.py b/app-instance/backend/tests/unit/test_filesystem_tools.py new file mode 100644 index 0000000..3199365 --- /dev/null +++ b/app-instance/backend/tests/unit/test_filesystem_tools.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path + +from beaver.tools import ObjectBackedTool, ToolContext +from beaver.tools.builtins import ListDirectoryTool, ReadFileTool, SearchFilesTool + + +def _run_tool(tool, arguments: dict, workspace: Path): + return asyncio.run( + ObjectBackedTool(tool).invoke(arguments, ToolContext(workspace=str(workspace))) + ) + + +def _payload(result): + return json.loads(result.content) + + +def test_list_directory_is_workspace_scoped(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "README.md").write_text("# Hello\n", encoding="utf-8") + (workspace / "src").mkdir() + + result = _run_tool(ListDirectoryTool(), {"path": "."}, workspace) + payload = _payload(result) + + assert result.success is True + assert payload["success"] is True + assert [entry["path"] for entry in payload["entries"]] == ["src", "README.md"] + + +def test_read_file_returns_limited_text(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "notes.txt").write_text("one\ntwo\nthree\n", encoding="utf-8") + + result = _run_tool(ReadFileTool(), {"path": "notes.txt", "start_line": 2, "max_lines": 1}, workspace) + payload = _payload(result) + + assert result.success is True + assert payload["success"] is True + assert payload["content"] == "two" + assert payload["start_line"] == 2 + assert payload["end_line"] == 2 + assert payload["truncated"] is True + + +def test_search_files_finds_paths_and_content(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "Dockerfile").write_text("FROM python:3.12\n", encoding="utf-8") + (workspace / "src").mkdir() + (workspace / "src" / "app.py").write_text("print('docker log')\n", encoding="utf-8") + + result = _run_tool(SearchFilesTool(), {"query": "docker", "max_results": 10}, workspace) + payload = _payload(result) + + assert result.success is True + assert payload["success"] is True + assert ("Dockerfile", "path") in { + (item["path"], item["match_type"]) for item in payload["results"] + } + assert ("src/app.py", "content") in { + (item["path"], item["match_type"]) for item in payload["results"] + } + + +def test_read_file_rejects_relative_path_escape(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + (tmp_path / "secret.txt").write_text("secret\n", encoding="utf-8") + + result = _run_tool(ReadFileTool(), {"path": "../secret.txt"}, workspace) + payload = _payload(result) + + assert result.success is False + assert payload["success"] is False + assert "escapes workspace" in payload["error"] + + +def test_read_file_rejects_absolute_path_escape(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("secret\n", encoding="utf-8") + + result = _run_tool(ReadFileTool(), {"path": str(outside)}, workspace) + payload = _payload(result) + + assert result.success is False + assert payload["success"] is False + assert "escapes workspace" in payload["error"] + + +def test_read_file_rejects_symlink_escape(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("secret\n", encoding="utf-8") + link = workspace / "outside-link.txt" + try: + os.symlink(outside, link) + except (OSError, NotImplementedError): + return + + result = _run_tool(ReadFileTool(), {"path": "outside-link.txt"}, workspace) + payload = _payload(result) + + assert result.success is False + assert payload["success"] is False + assert "escapes workspace" in payload["error"] + + +def test_read_file_rejects_binary_files(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "blob.bin").write_bytes(b"abc\x00def") + + result = _run_tool(ReadFileTool(), {"path": "blob.bin"}, workspace) + payload = _payload(result) + + assert result.success is False + assert payload["success"] is False + assert "binary" in payload["error"] + diff --git a/app-instance/backend/tests/unit/test_gateway_channels.py b/app-instance/backend/tests/unit/test_gateway_channels.py new file mode 100644 index 0000000..76a852f --- /dev/null +++ b/app-instance/backend/tests/unit/test_gateway_channels.py @@ -0,0 +1,201 @@ +import asyncio +from dataclasses import dataclass, field +from typing import Any + +from beaver.foundation.events import InboundMessage, MessageBus +from beaver.interfaces.channels import ChannelManager, MemoryChannelAdapter +from beaver.interfaces.gateway.main import run_gateway +from beaver.services.agent_service import AgentService + + +@dataclass(slots=True) +class FakeResult: + session_id: str + run_id: str = "run-1" + output_text: str = "" + finish_reason: str = "stop" + provider_name: str | None = "fake" + model: str | None = "fake-model" + usage: dict[str, Any] = field(default_factory=dict) + + +class FakeService: + is_running = True + + async def submit_direct(self, message: str, **kwargs: Any) -> FakeResult: + return FakeResult( + session_id=kwargs.get("session_id") or "s1", + output_text=f"echo:{message}", + ) + + async def handle_inbound_message(self, inbound: InboundMessage): + result = await self.submit_direct(inbound.content, session_id=inbound.session_id) + return AgentService.build_outbound_message(inbound, result) + + +class SlowService: + is_running = True + + async def submit_direct(self, message: str, **kwargs: Any) -> FakeResult: + await asyncio.sleep(10) + return FakeResult(session_id=kwargs.get("session_id") or "s1") + + async def handle_inbound_message(self, inbound: InboundMessage): + result = await self.submit_direct(inbound.content, session_id=inbound.session_id) + return AgentService.build_outbound_message(inbound, result) + + +def test_gateway_routes_memory_channel_roundtrip() -> None: + async def run() -> None: + bus = MessageBus() + channel = MemoryChannelAdapter(bus) + stop_event = asyncio.Event() + task = asyncio.create_task( + run_gateway( + service=FakeService(), + manage_service_lifecycle=False, + bus=bus, + channels=[channel], + stop_event=stop_event, + ) + ) + + await channel.publish_text("hello", session_id="s1") + for _ in range(40): + if channel.sent_messages: + break + await asyncio.sleep(0.05) + + assert channel.sent_messages + message = channel.sent_messages[0] + assert message.content == "echo:hello" + assert message.session_id == "s1" + assert message.finish_reason == "stop" + + stop_event.set() + await asyncio.wait_for(task, timeout=2) + + asyncio.run(run()) + + +def test_gateway_delivers_cancelled_outbound_to_channel() -> None: + async def run() -> None: + bus = MessageBus() + channel = MemoryChannelAdapter(bus) + stop_event = asyncio.Event() + task = asyncio.create_task( + run_gateway( + service=SlowService(), + manage_service_lifecycle=False, + bus=bus, + channels=[channel], + stop_event=stop_event, + ) + ) + + await channel.publish_text("slow", session_id="s1") + await asyncio.sleep(0.05) + stop_event.set() + await asyncio.wait_for(task, timeout=3) + + assert channel.sent_messages + assert channel.sent_messages[0].finish_reason == "cancelled" + + asyncio.run(run()) + + +def test_gateway_rejects_channel_manager_and_channels_together() -> None: + async def run() -> None: + bus = MessageBus() + try: + await run_gateway( + service=FakeService(), + manage_service_lifecycle=False, + bus=bus, + channel_manager=ChannelManager(bus), + channels=[MemoryChannelAdapter(bus)], + stop_event=asyncio.Event(), + ) + except ValueError as exc: + assert "either channel_manager or channels" in str(exc) + else: + raise AssertionError("expected ValueError") + + asyncio.run(run()) + + +def test_agent_service_maps_inbound_error_to_structured_outbound() -> None: + async def run() -> None: + service = AgentService() + + async def failing_submit_direct(message: str, **kwargs: Any) -> FakeResult: + raise RuntimeError("boom") + + service.submit_direct = failing_submit_direct # type: ignore[method-assign] + outbound = await service.handle_inbound_message( + InboundMessage(channel="memory", content="hello", session_id="s1", metadata={"source": "test"}) + ) + + assert outbound.finish_reason == "error" + assert outbound.session_id == "s1" + assert outbound.metadata["error"] == "boom" + assert outbound.metadata["inbound_metadata"] == {"source": "test"} + + asyncio.run(run()) + + +def test_channel_manager_start_cancellation_rolls_back_started_channels() -> None: + class StartedChannel: + name = "started" + + def __init__(self, bus: MessageBus) -> None: + self.bus = bus + self.stopped = False + + async def start(self) -> None: + pass + + async def stop(self) -> None: + self.stopped = True + + async def send(self, message: Any) -> None: + pass + + class BlockingChannel: + name = "blocking" + + def __init__(self, bus: MessageBus) -> None: + self.bus = bus + self.entered = asyncio.Event() + + async def start(self) -> None: + self.entered.set() + await asyncio.sleep(10) + + async def stop(self) -> None: + pass + + async def send(self, message: Any) -> None: + pass + + async def run() -> None: + bus = MessageBus() + started = StartedChannel(bus) + blocking = BlockingChannel(bus) + manager = ChannelManager(bus) + manager.register(started) + manager.register(blocking) + + task = asyncio.create_task(manager.start()) + await blocking.entered.wait() + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + else: + raise AssertionError("expected cancellation") + + assert started.stopped + + asyncio.run(run()) diff --git a/app-instance/backend/tests/unit/test_imports.py b/app-instance/backend/tests/unit/test_imports.py index 1b80785..b0eef33 100644 --- a/app-instance/backend/tests/unit/test_imports.py +++ b/app-instance/backend/tests/unit/test_imports.py @@ -1,12 +1,13 @@ -from beaver.engine import AgentLoop +from beaver.engine import AgentLoop, EngineLoader from beaver.foundation.events import InboundMessage, MessageBus, OutboundMessage +from beaver.interfaces.channels import ChannelManager, MemoryChannelAdapter from beaver.interfaces.gateway.main import run_gateway from beaver.interfaces.web.app import create_app from beaver.interfaces.web.schemas import WebChatRequest, WebChatResponse -def test_agent_loop_boots() -> None: - loop = AgentLoop() +def test_agent_loop_boots(tmp_path) -> None: + loop = AgentLoop(loader=EngineLoader(workspace=tmp_path)) loaded = loop.boot() assert "echo" in loaded.tools assert "memory" in loaded.tools @@ -29,6 +30,14 @@ def test_message_bus_imports() -> None: assert OutboundMessage(channel="test", content="ok", session_id=None, finish_reason="stop").content == "ok" +def test_channel_imports() -> None: + bus = MessageBus() + channel = MemoryChannelAdapter(bus) + manager = ChannelManager(bus) + manager.register(channel) + assert manager.channels["memory"] is channel + + def test_web_schema_imports() -> None: assert WebChatRequest(message="hello").message == "hello" assert WebChatResponse( diff --git a/app-instance/backend/tests/unit/test_tool_assembler.py b/app-instance/backend/tests/unit/test_tool_assembler.py new file mode 100644 index 0000000..3eabe48 --- /dev/null +++ b/app-instance/backend/tests/unit/test_tool_assembler.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +from beaver.engine.context import SkillContext +from beaver.foundation.embedding import EmbeddingRetriever +from beaver.skills.catalog.loader import SkillsLoader +from beaver.tools import BaseTool, ToolAssembler, ToolContext, ToolExecutor, ToolRegistry, ToolResult, ToolSpec + + +class DummyTool(BaseTool): + def __init__( + self, + name: str, + *, + description: str | None = None, + toolset: str = "test", + always_available: bool = False, + ) -> None: + self._spec = ToolSpec( + name=name, + description=description or name, + input_schema={"type": "object", "properties": {}}, + toolset=toolset, + always_available=always_available, + ) + + @property + def spec(self) -> ToolSpec: + return self._spec + + async def invoke(self, arguments: dict, context: ToolContext) -> ToolResult: + return ToolResult(success=True, content="ok", tool_name=self.spec.name) + + +class StaticRetriever: + async def retrieve(self, **kwargs): + candidates = kwargs["candidates"] + top_k = kwargs["top_k"] + preferred = ["search_files", "echo"] + ordered = sorted( + candidates, + key=lambda item: preferred.index(item["name"]) if item["name"] in preferred else len(preferred), + ) + return ordered[:top_k] + + +def test_tool_spec_exports_mcp_and_provider_schema() -> None: + spec = ToolSpec( + name="read_file", + description="Read a file", + input_schema={"type": "object", "properties": {"path": {"type": "string"}}}, + toolset="file", + ) + + assert spec.to_mcp_descriptor() == { + "name": "read_file", + "description": "Read a file", + "inputSchema": {"type": "object", "properties": {"path": {"type": "string"}}}, + } + assert spec.to_provider_schema()["function"]["parameters"] == spec.input_schema + + +def test_tool_assembler_merges_always_skill_hints_and_embedding(tmp_path: Path) -> None: + skill_dir = tmp_path / "skills" / "docker-debug" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + """--- +name: docker-debug +description: Debug Docker issues. +tools: + - terminal +--- + +# Docker Debug +""", + encoding="utf-8", + ) + + registry = ToolRegistry() + registry.register(DummyTool("memory", toolset="memory", always_available=True)) + registry.register(DummyTool("skill_view", toolset="skills", always_available=True)) + registry.register(DummyTool("terminal", toolset="shell")) + registry.register(DummyTool("search_files", toolset="file")) + registry.register(DummyTool("echo", toolset="debug")) + + assembler = ToolAssembler(retriever=StaticRetriever()) + loader = SkillsLoader(tmp_path) + selected = asyncio.run( + assembler.assemble( + task_description="排查 Docker 容器日志", + registry=registry, + skills_loader=loader, + activated_skills=[SkillContext(name="docker-debug", content="")], + top_k=1, + ) + ) + + assert [spec.name for spec in selected] == ["memory", "skill_view", "terminal", "search_files"] + + +def test_embedding_fallback_can_return_all_or_top_k() -> None: + candidates = [{"name": f"tool_{index}", "description": "", "input_schema": "{}"} for index in range(3)] + retriever = EmbeddingRetriever(api_key_env="MISSING_EMBEDDING_KEY", api_base_env="MISSING_EMBEDDING_BASE") + + all_candidates = asyncio.run( + retriever.retrieve(query="x", candidates=candidates, top_k=1, fallback_top_k=None) + ) + top_candidate = asyncio.run( + retriever.retrieve(query="x", candidates=candidates, top_k=1, fallback_top_k=1) + ) + + assert [item["name"] for item in all_candidates] == ["tool_0", "tool_1", "tool_2"] + assert [item["name"] for item in top_candidate] == ["tool_0"] + + +def test_beaver_tools_import_does_not_load_provider_stack_with_socks_proxy() -> None: + code = ( + "import beaver.tools\n" + "from beaver.skills.catalog.loader import SkillsLoader\n" + "print('ok')" + ) + result = subprocess.run( + [sys.executable, "-c", code], + check=False, + capture_output=True, + text=True, + env={ + "PYTHONPATH": str(Path(__file__).resolve().parents[2]), + "HTTP_PROXY": "socks://127.0.0.1:7897/", + "HTTPS_PROXY": "socks://127.0.0.1:7897/", + }, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "ok" + + +def test_tool_executor_parses_object_tool_call_string_arguments() -> None: + tool_call = SimpleNamespace(name="echo", arguments='{"text": "hello"}') + + name, arguments = ToolExecutor._normalize_tool_call(tool_call) + + assert name == "echo" + assert arguments == {"text": "hello"} diff --git a/app-instance/backend/施工指南.md b/app-instance/backend/施工指南.md index 3f60571..913dec4 100644 --- a/app-instance/backend/施工指南.md +++ b/app-instance/backend/施工指南.md @@ -704,6 +704,7 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 1. `beaver/tools/base.py` 2. `beaver/tools/registry/tool_registry.py` +3. `beaver/tools/assembler/task_assembler.py` 参考旧文件： @@ -732,6 +733,48 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 1. registry 可以注册工具 2. provider 返回 tool call 时可以找到并执行工具 3. memory / session_search 已纳入统一工具集合 +4. 本地工具描述采用 MCP-style `name/description/inputSchema` +5. `ToolAssembler` 能按 task description 用 embedding 召回本轮 top10 工具 +6. activated skill frontmatter 里的 `tools` 能参与本轮工具选择 +7. 只读 filesystem tools 已接入： + - `list_directory` + - `read_file` + - `search_files` +8. filesystem tools 强制限制在 `ToolContext.workspace` +9. 相对路径逃逸、绝对路径逃逸、符号链接逃逸都会拒绝 +10. 二进制文件读取会拒绝，搜索会跳过二进制 / 大文件 + +当前工具选择规则已经定为： + +1. always tools 每轮默认可用 + - `memory` + - `session_search` + - `skill_view` + - `list_directory` + - `read_file` + - `search_files` +2. activated skill 可以显式声明工具： + +```yaml +--- +tools: + - terminal + - read_file +--- +``` + +3. `ToolAssembler` 合并： + - always tools + - skill hints + - task embedding top10 tools +4. 第一版只信任 frontmatter / metadata 的显式 `tools`，不从正文里猜工具名 +5. 如果 skill 声明了尚未注册的工具，先忽略，不阻断 run + +filesystem 这一版只做只读，不做写文件 / shell： + +1. 先让 agent 能看见 workspace 结构、读源码、搜文本 +2. 写文件和 shell 属于高风险工具，必须等 permission gates 明确后再接 +3. 当前 workspace 边界只保证路径隔离，不等价于完整权限系统 ### 4.5 最后实现第一版 `AgentLoop` @@ -855,6 +898,23 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 - 先用 embedding 做语义召回 - 输出应该激活的 skills 6. embedding 配置通过 provider bundle 的独立 `embedding runtime` 传入；若没有显式 embedding 配置，则只有主链本身是 OpenAI-compatible 时才允许继承 `api_base/api_key` +7. skill frontmatter 可声明本 skill 推荐工具；这些 tool hints 会交给 `ToolAssembler` + +当前和长期目标的关系： + +1. 已完成基础入口： + - curated memory CRUD + - `session_search` + - `skill_view` + - `SkillAssembler` + - `ToolAssembler` +2. 还没完成长期智能体治理： + - 智能体定期整理 / 提示记忆 + - 复杂任务完成后自主创建技能 + - 技能在使用过程中自我提升 + - FTS5 + LLM 摘要的跨会话回忆增强 + - Honcho 风格辩证用户建模 + - agentskills.io 开放标准兼容 --- @@ -884,6 +944,10 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 3. `ContextBuilder` - 不再直接依赖进程内状态 - 只从 Session / curated memory / skills 中提取当前需要的上下文 +4. `ToolAssembler` + - 不把所有工具无脑暴露给模型 + - 按 task description / activated skill hints 选择本轮工具 + - 输出 provider 可消费的 tool schema 这一步不是要一口气做完 fork / rewind / checkpoint 全套系统，而是先把“Session-first, Stateless Harness”这条主线立住。 @@ -926,12 +990,13 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 1. `session_started/ensured` 2. `run_started` -3. `system_prompt_snapshotted` -4. `user_message_added` -5. `assistant_message_added` -6. `tool_call_requested` -7. `tool_result_recorded` -8. `run_failed` 或 `run_completed` +3. `skill_activation_snapshotted` +4. `tool_selection_snapshotted` +5. `system_prompt_snapshotted` +6. `user_message_added` +7. `assistant_message_added` +8. `tool_result_recorded` +9. `run_failed` 或 `run_completed` 并且每次 run 都要带独立 `run_id`，这样同一个 session 内的多次运行才能被切开。 @@ -1016,6 +1081,31 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 7. `stop()` / `shutdown()` 应支持 graceful timeout，必要时允许 force cancel 8. `close()`：只有在实例已停止后才能释放 runtime 资源 +这一阶段也明确了模型配置的归属： + +1. 大模型 provider / api_key / api_base 属于 backend runtime config +2. Web / Gateway / Channel 不单独保存模型密钥 +3. 前端请求不传 API Key，只传 `message/session_id/user_id` 等业务输入 +4. Docker 单实例部署时，每个用户 sandbox 读取自己的实例配置 +5. 默认使用新 Beaver 实例目录： + - `/root/.beaver/config.json` + - `/root/.beaver/workspace` +6. 新 Beaver 命名优先使用： + - `BEAVER_CONFIG_PATH` + - `BEAVER_HOME/config.json` +7. 兼容迁移期旧命名： + - `NANOBOT_CONFIG_PATH` + - `NANOBOT_HOME/config.json` + +app-instance 镜像也已经切到新 Beaver 后端： + +1. Dockerfile 只安装 `backend/beaver` +2. 不再复制旧 `backend/nanobot`、`backend/bridge`、vendored `swarms` +3. entrypoint 通过 `python -m uvicorn beaver.interfaces.web.app:create_app --factory` 启动 Web +4. 容器内默认配置与 workspace 使用： + - `/root/.beaver/config.json` + - `/root/.beaver/workspace` + ### 6.2.1 Web / Gateway 现在如何接这套 lifecycle 这一层现在已经开始落成真正的宿主层，而不是只停留在文档占位： @@ -1045,13 +1135,25 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 - `run_gateway()` 启动时： - 如果 gateway 自己创建 service，则 `await service.start()` - 持有最小 `MessageBus` - - 常驻消费 `bus.inbound` - - 调 `await service.submit_direct(...)` - - 将结果写回 `bus.outbound` + - 可选接收 `ChannelManager` / channel adapters + - `ChannelManager` 和 `channels` 参数二选一： + - 传 `ChannelManager`：外部提前配置好 channel + - 传 `channels`：gateway 内部创建 `ChannelManager` 并注册这些 channel + - inbound 流向： + - channel adapter 发布 `InboundMessage` + - `MessageBus.inbound` + - gateway bridge 常驻消费 + - 调 `await service.handle_inbound_message(...)` + - outbound 流向： + - `AgentService` 内部完成 `InboundMessage -> OutboundMessage` 映射 + - gateway bridge 将结果写回 `MessageBus.outbound` + - 如果启用了 `ChannelManager`，则分发给对应 channel adapter + - 未启用 `ChannelManager` 时，保留直接消费 `bus.outbound` 的最小测试能力 - 同时等待 `stop_event` - 停机时： - 先尝试 `await service.shutdown(timeout_seconds=5.0, force=True)` - 再等待 bridge 协程收尾；必要时取消 bridge + - 再等待 outbound dispatch 协程收尾；必要时取消 dispatch - 如果 gateway 自己接管 lifecycle 且 `start()` 失败： - 立即 `close()` 做 startup cleanup - 未处理完的 inbound： @@ -1068,6 +1170,16 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 - `outbound` - 还没有 broker / topic routing / retry / persistence +4. `beaver/interfaces/channels/*` + - 已经补了最小 channel adapter 层： + - `ChannelAdapter` + - `ChannelManager` + - `MemoryChannelAdapter` + - 当前 channel 职责很窄： + - 把外部输入发布成 `InboundMessage` + - 接收并投递 `OutboundMessage` + - `MemoryChannelAdapter` 只用于本地测试和内嵌接入，不是正式消息 broker + 所以现在已经明确： 1. Web / Gateway 属于宿主层 @@ -1082,16 +1194,18 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 - 外部注入的 `AgentService`：默认不自动 start/shutdown，除非显式要求接管 5. gateway 已经从“只会常驻等待”推进到“最小消息桥接层” - external inbound message + - channel adapter - `MessageBus.inbound` - - `service.submit_direct(...)` + - `service.handle_inbound_message(...)` - `MessageBus.outbound` + - channel adapter outbound delivery 但这一阶段还没做： -1. channels adapter -2. realtime streaming -3. platform-level supervisor -4. 更复杂的 bus 语义（retry / routing / persistence） +1. realtime streaming +2. platform-level supervisor +3. 更复杂的 bus 语义（retry / routing / persistence） +4. 外部真实 channel adapter ### 6.3 第三步：回填 bus 模式 @@ -1107,9 +1221,16 @@ provider 层不要再做成“一个厂商一个世界”，而是要拆成 4 需要补的函数： 1. 从 inbound 读取消息 -2. 调用 `_process_message` +2. 通过 `AgentService.handle_inbound_message(...)` 映射到 runtime 调用 3. 发布 outbound +当前这一步的最小收口方式已经确定为： + +1. `MessageBus` 只负责协议和队列 +2. `gateway` 只负责宿主、常驻消费和 channel 分发 +3. `InboundMessage -> AgentRunResult -> OutboundMessage` 的映射收口在 `AgentService` +4. `AgentLoop` 继续只关心 agent 执行内核，不直接感知 bus 协议 + 注意： 只有在 `process_direct()` 稳定，并且 6.1 / 6.2 已经把 Session-first + lifecycle 骨架立住后，才做 `run()` 的长循环版本。 diff --git a/app-instance/create-instance.sh b/app-instance/create-instance.sh index a7e2b6a..118142b 100755 --- a/app-instance/create-instance.sh +++ b/app-instance/create-instance.sh @@ -194,7 +194,7 @@ if outlook_mcp_url: data = { "agents": { "defaults": { - "workspace": "/root/.nanobot/workspace", + "workspace": "/root/.beaver/workspace", "model": os.environ["MODEL"], } }, @@ -505,13 +505,13 @@ PY fi INSTANCE_ROOT="${INSTANCES_ROOT}/${INSTANCE_SLUG}" -NANOBOT_HOME="${INSTANCE_ROOT}/nanobot-home" -CONFIG_PATH="${NANOBOT_HOME}/config.json" -AUTH_USERS_PATH="${NANOBOT_HOME}/web_auth_users.json" -RUNTIME_ENV_PATH="${NANOBOT_HOME}/runtime.env" -WORKSPACE_PATH="${NANOBOT_HOME}/workspace" +BEAVER_HOME="${INSTANCE_ROOT}/beaver-home" +CONFIG_PATH="${BEAVER_HOME}/config.json" +AUTH_USERS_PATH="${BEAVER_HOME}/web_auth_users.json" +RUNTIME_ENV_PATH="${BEAVER_HOME}/runtime.env" +WORKSPACE_PATH="${BEAVER_HOME}/workspace" -mkdir -p "$NANOBOT_HOME" "$WORKSPACE_PATH" +mkdir -p "$BEAVER_HOME" "$WORKSPACE_PATH" render_config_json "$CONFIG_PATH" render_auth_users_json "$AUTH_USERS_PATH" @@ -540,8 +540,12 @@ RUN_ARGS=( --name "$CONTAINER_NAME" --restart unless-stopped -p "${HOST_BIND_IP}:${HOST_PORT}:8080" - -v "${NANOBOT_HOME}:/root/.nanobot" - -e "NANOBOT_AUTH_FILE=/root/.nanobot/web_auth_users.json" + -v "${BEAVER_HOME}:/root/.beaver" + -e "BEAVER_HOME=/root/.beaver" + -e "BEAVER_CONFIG_PATH=/root/.beaver/config.json" + -e "BEAVER_WORKSPACE=/root/.beaver/workspace" + -e "NANOBOT_HOME=/root/.beaver" + -e "NANOBOT_AUTH_FILE=/root/.beaver/web_auth_users.json" -e "NANOBOT_FRONTEND_PUBLIC_BASE_URL=${PUBLIC_URL}" -e "APP_PUBLIC_PORT=8080" -e "APP_FRONTEND_PORT=3000" @@ -567,7 +571,7 @@ docker run "${RUN_ARGS[@]}" "$IMAGE_NAME" >/dev/null --host-port "$HOST_PORT" \ --public-url "$PUBLIC_URL" \ --instance-root "$INSTANCE_ROOT" \ - --nanobot-home "$NANOBOT_HOME" \ + --nanobot-home "$BEAVER_HOME" \ --config-path "$CONFIG_PATH" \ --auth-users-path "$AUTH_USERS_PATH" \ --network-name "$NETWORK_NAME" \ @@ -589,7 +593,8 @@ image_name=${IMAGE_NAME} host_port=${HOST_PORT} public_url=${PUBLIC_URL} instance_root=${INSTANCE_ROOT} -nanobot_home=${NANOBOT_HOME} +beaver_home=${BEAVER_HOME} +nanobot_home=${BEAVER_HOME} config_path=${CONFIG_PATH} auth_users_path=${AUTH_USERS_PATH} runtime_env_path=${RUNTIME_ENV_PATH} diff --git a/app-instance/entrypoint.sh b/app-instance/entrypoint.sh index f7e7083..385ce08 100755 --- a/app-instance/entrypoint.sh +++ b/app-instance/entrypoint.sh @@ -4,9 +4,12 @@ set -euo pipefail APP_PUBLIC_PORT="${APP_PUBLIC_PORT:-8080}" APP_FRONTEND_PORT="${APP_FRONTEND_PORT:-3000}" APP_BACKEND_PORT="${APP_BACKEND_PORT:-18080}" -NANOBOT_HOME="${NANOBOT_HOME:-/root/.nanobot}" -NANOBOT_AUTH_FILE="${NANOBOT_AUTH_FILE:-$NANOBOT_HOME/web_auth_users.json}" -NANOBOT_RUNTIME_ENV_FILE="${NANOBOT_RUNTIME_ENV_FILE:-$NANOBOT_HOME/runtime.env}" +BEAVER_HOME="${BEAVER_HOME:-/root/.beaver}" +BEAVER_CONFIG_PATH="${BEAVER_CONFIG_PATH:-$BEAVER_HOME/config.json}" +BEAVER_WORKSPACE="${BEAVER_WORKSPACE:-$BEAVER_HOME/workspace}" +NANOBOT_HOME="${NANOBOT_HOME:-$BEAVER_HOME}" +NANOBOT_AUTH_FILE="${NANOBOT_AUTH_FILE:-$BEAVER_HOME/web_auth_users.json}" +NANOBOT_RUNTIME_ENV_FILE="${NANOBOT_RUNTIME_ENV_FILE:-$BEAVER_HOME/runtime.env}" log() { printf '[app-instance] %s\n' "$*" @@ -21,40 +24,6 @@ require_file() { fi } -render_swarms_env_file() { - local config_path="$1" - local target_path="$2" - - CONFIG_PATH="$config_path" TARGET_PATH="$target_path" python3 - <<'PY' -import json -import os -from pathlib import Path - -config_path = Path(os.environ["CONFIG_PATH"]) -target_path = Path(os.environ["TARGET_PATH"]) - -data = json.loads(config_path.read_text(encoding="utf-8")) -model = str(data.get("agents", {}).get("defaults", {}).get("model") or "").strip() -if model and "/" not in model: - model = f"openai/{model}" -provider_cfg = data.get("providers", {}).get("openai", {}) or {} -api_key = str(provider_cfg.get("apiKey") or "").strip() -api_base = str(provider_cfg.get("apiBase") or "").strip() - -lines = [ - '# Generated from /root/.nanobot/config.json for vendored swarms runtime.', - 'WORKSPACE_DIR="/root/.nanobot/workspace"', - 'SWARMS_VERBOSE_GLOBAL="False"', - 'SWARMS_TELEMETRY_ON="false"', - f'SWARMS_DEFAULT_MODEL="{model}"', - f'OPENAI_API_KEY="{api_key}"', - f'OPENAI_API_BASE="{api_base}"', - f'OPENAI_BASE_URL="{api_base}"', -] -target_path.write_text("\n".join(lines) + "\n", encoding="utf-8") -PY -} - cleanup() { local status=$? @@ -74,7 +43,7 @@ cleanup() { trap cleanup EXIT INT TERM -mkdir -p "$NANOBOT_HOME" "$NANOBOT_HOME/workspace" +mkdir -p "$BEAVER_HOME" "$BEAVER_WORKSPACE" if [[ -f "$NANOBOT_RUNTIME_ENV_FILE" ]]; then set -a @@ -82,24 +51,21 @@ if [[ -f "$NANOBOT_RUNTIME_ENV_FILE" ]]; then set +a fi -require_file "$NANOBOT_HOME/config.json" "Missing Boardware Genius config" -require_file "$NANOBOT_AUTH_FILE" "Missing web auth users file" - -SWARMS_ENV_FILE="/opt/app/backend/third_party/swarms/.env" -render_swarms_env_file "$NANOBOT_HOME/config.json" "$SWARMS_ENV_FILE" -if [[ -f "$SWARMS_ENV_FILE" ]]; then - set -a - . "$SWARMS_ENV_FILE" - set +a -fi +require_file "$BEAVER_CONFIG_PATH" "Missing Beaver config" export NANOBOT_AUTH_FILE export NANOBOT_RUNTIME_ENV_FILE +export BEAVER_HOME +export BEAVER_CONFIG_PATH +export BEAVER_WORKSPACE export PORT="$APP_FRONTEND_PORT" export HOSTNAME="127.0.0.1" -log "starting backend on 127.0.0.1:${APP_BACKEND_PORT}" -nanobot web --host 127.0.0.1 --port "$APP_BACKEND_PORT" & +log "starting Beaver backend on 127.0.0.1:${APP_BACKEND_PORT}" +( + cd /opt/app/backend + python -m uvicorn "beaver.interfaces.web.app:create_app" --factory --host 127.0.0.1 --port "$APP_BACKEND_PORT" +) & BACKEND_PID=$! log "starting frontend on 127.0.0.1:${APP_FRONTEND_PORT}" diff --git a/app-instance/frontend/lib/api.ts b/app-instance/frontend/lib/api.ts index c7b408c..4b090cc 100644 --- a/app-instance/frontend/lib/api.ts +++ b/app-instance/frontend/lib/api.ts @@ -251,10 +251,21 @@ export async function sendMessage( if (attachments && attachments.length > 0) { body.attachments = attachments; } - return fetchJSON('/api/chat', { + const result = await fetchJSON<{ + response?: string; + status?: string; + session_id: string; + output_text?: string; + finish_reason?: string; + }>('/api/chat', { method: 'POST', body: JSON.stringify(body), }); + return { + response: result.response ?? result.output_text, + status: result.status ?? result.finish_reason, + session_id: result.session_id, + }; } export function streamMessage( diff --git a/deploy-control/server.py b/deploy-control/server.py index 72ebbd2..c66cbf1 100755 --- a/deploy-control/server.py +++ b/deploy-control/server.py @@ -183,7 +183,7 @@ def wait_for_backend(record: dict[str, Any]) -> None: try: with urllib_request.urlopen(target, timeout=5) as response: payload = json.loads(response.read().decode("utf-8")) - if payload.get("message") == "pong": + if payload.get("message") == "pong" or payload.get("status") == "ok": return last_error = f"unexpected ping response from {target}" except (urllib_error.URLError, TimeoutError, json.JSONDecodeError) as exc: