From 269661afff317d011e7c9f92d6ea4723dffd36f8 Mon Sep 17 00:00:00 2001 From: tomtan Date: Tue, 16 Jun 2026 13:36:18 +0800 Subject: [PATCH] =?UTF-8?q?feat(memory-gateway):=20=E5=BC=95=E5=85=A5=20Me?= =?UTF-8?q?mory=20Gateway=20=E9=85=8D=E7=BD=AE=E3=80=81=E5=87=AD=E6=8D=AE?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=92=8C=E6=9C=8D=E5=8A=A1=E7=BC=96=E6=8E=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 新增 MemoryGatewayConfig 和 MemoryConfig dataclass，用于配置管理。 * 实现 MemoryGatewayUserCredential 和 MemoryGatewayCredentialStore，用于处理用户凭据。 * 创建 MemoryGatewayService，用于管理与 Memory Gateway 的交互。 * 开发用于记忆设置的 JSON 配置文件。 * 增强单元测试，覆盖新功能，包括凭据存储和服务行为。 * 更新 entrypoint 和实例创建脚本，以初始化 Memory Gateway 用户存储。 --- app-instance/Dockerfile | 1 + app-instance/README.md | 13 ++ app-instance/backend/README.md | 44 +++++-- app-instance/backend/beaver/engine/loader.py | 54 ++++++-- app-instance/backend/beaver/engine/loop.py | 11 +- .../beaver/foundation/config/__init__.py | 3 +- .../beaver/foundation/config/loader.py | 65 +++++---- .../beaver/foundation/config/schema.py | 29 +---- .../integrations/memory_gateway/__init__.py | 5 - .../backend/beaver/interfaces/web/app.py | 64 ++++++++- .../backend/beaver/memory/gateway/__init__.py | 23 ++++ .../gateway}/client.py | 7 +- .../backend/beaver/memory/gateway/config.py | 32 +++++ .../beaver/memory/gateway/credentials.py | 75 +++++++++++ .../gateway/service.py} | 17 ++- .../backend/beaver/services/__init__.py | 6 +- app-instance/backend/memory/config.json | 13 ++ .../backend/tests/unit/test_config_loader.py | 113 ++++++++++------ .../unit/test_memory_gateway_agent_loop.py | 61 +++++++-- .../unit/test_memory_gateway_credentials.py | 58 +++++++++ .../tests/unit/test_memory_gateway_loader.py | 26 ++-- .../unit/test_memory_gateway_registration.py | 123 ++++++++++++++++++ .../tests/unit/test_memory_gateway_service.py | 25 ++-- .../backend/tests/unit/test_websocket_chat.py | 69 ++++++++++ app-instance/create-instance.sh | 4 + app-instance/entrypoint.sh | 7 + 26 files changed, 788 insertions(+), 160 deletions(-) delete mode 100644 app-instance/backend/beaver/integrations/memory_gateway/__init__.py create mode 100644 app-instance/backend/beaver/memory/gateway/__init__.py rename app-instance/backend/beaver/{integrations/memory_gateway => memory/gateway}/client.py (89%) create mode 100644 app-instance/backend/beaver/memory/gateway/config.py create mode 100644 app-instance/backend/beaver/memory/gateway/credentials.py rename app-instance/backend/beaver/{services/memory_gateway_service.py => memory/gateway/service.py} (88%) create mode 100644 app-instance/backend/memory/config.json create mode 100644 app-instance/backend/tests/unit/test_memory_gateway_credentials.py create mode 100644 app-instance/backend/tests/unit/test_memory_gateway_registration.py diff --git a/app-instance/Dockerfile b/app-instance/Dockerfile index 0d4ec10..609cad3 100644 --- a/app-instance/Dockerfile +++ b/app-instance/Dockerfile @@ -67,6 +67,7 @@ WORKDIR /opt/app/backend COPY backend/pyproject.toml backend/README.md ./ COPY backend/beaver/ ./beaver/ +COPY backend/memory/ ./memory/ RUN uv pip install --system --no-cache --index-url "${PYPI_INDEX_URL}" ".[channels]" WORKDIR /opt/app/frontend diff --git a/app-instance/README.md b/app-instance/README.md index 36c844d..7723534 100644 --- a/app-instance/README.md +++ b/app-instance/README.md @@ -110,6 +110,8 @@ runtime/instances// runtime/instances// └── beaver-home ├── config.json + ├── memory_gateway_users.json + ├── runtime.env ├── web_auth_users.json └── workspace/ ``` @@ -125,10 +127,21 @@ runtime/instances// ```text BEAVER_CONFIG_PATH=/root/.beaver/config.json BEAVER_WORKSPACE=/root/.beaver/workspace +BEAVER_MEMORY_GATEWAY_USERS_PATH=/root/.beaver/memory_gateway_users.json ``` 所以模型 `provider/api_key/api_base/model` 配一次即可，Web / channel 请求不需要、也不应该携带 API Key。 +Memory Gateway 的共享非密钥配置不放在实例目录里，而是放在仓库内的： + +```text +app-instance/backend/memory/config.json +``` + +实例目录只保存按 Beaver 登录用户名分组的 Gateway 凭证。`create-instance.sh` +会初始化空的 `memory_gateway_users.json`，容器启动时也会兜底创建这个文件并设置 +`0600` 权限。 + `create-instance.sh` 默认会把仓库根目录的 `skills/` 非覆盖式复制到实例 workspace，并把同一个目录只读挂载到实例容器的 `/opt/app/initial-skills`。`entrypoint.sh` 每次启动都会用该目录补齐缺失的 published 初始 skills；已有 skill 目录不会被覆盖，index 只做并集追加。 ## 当前状态 diff --git a/app-instance/backend/README.md b/app-instance/backend/README.md index e58cc64..e5d35ab 100644 --- a/app-instance/backend/README.md +++ b/app-instance/backend/README.md @@ -35,19 +35,23 @@ Curated memory 始终启用：每轮仍会冻结并注入 `MEMORY.md` / `USER.md 每轮先调用 `/memories/search`，正常完成后调用一次 `/memories/add`，成功后再调用 一次 `/memories/flush`。两套存储不会互相同步、覆盖或去重。 -完整配置示例： +共享 Gateway 配置放在： + +```text +app-instance/backend/memory/config.json +``` + +当前默认内容： ```json { "memory": { "mode": "hybrid", "gateway": { - "baseUrl": "http://127.0.0.1:8010", - "userId": "gateway_test_user", - "userKey": "uk_xxx", + "baseUrl": "http://172.19.207.37:8010", "appId": "default", "projectId": "default", - "scope": ["current_chat", "resources"], + "scope": ["current_chat", "resources", "all_user_memory"], "topK": 8, "timeoutSeconds": 10 } @@ -55,10 +59,28 @@ Curated memory 始终启用：每轮仍会冻结并注入 `MEMORY.md` / `USER.md } ``` -- `memory` 整段缺失时，默认采用隐式 `hybrid`；Gateway 凭证不完整会告警并只运行 curated memory。 -- 显式配置 `"mode": "hybrid"` 时，`baseUrl`、`userId` 和 `userKey` 缺失会导致启动失败。 -- 配置 `"mode": "curated"` 可关闭 Gateway，curated memory 行为不变。 +每个实例自己的 Gateway 用户凭证放在： + +```text +/root/.beaver/memory_gateway_users.json +``` + +格式示例： + +```json +{ + "users": { + "tom": { + "userId": "tom", + "userKey": "uk_xxx" + } + } +} +``` + +- 前端 `POST /api/auth/register` 会用 Beaver 登录用户名调用 Gateway `POST /users`，并把返回的 `userId/userKey` 写入实例凭证文件。 +- REST `/api/chat` 和 WebSocket `/ws/...` 只使用登录 token 解析出的 Beaver 用户名来选择 Gateway 凭证，请求体里的 `user_id` 不参与 Gateway 身份选择。 +- 某个登录用户还没有 Gateway 凭证时，这一轮只走 curated memory，不会报 chat 级错误。 +- `BEAVER_MEMORY_CONFIG_PATH` 可覆盖共享 memory 配置路径，`BEAVER_MEMORY_GATEWAY_USERS_PATH` 可覆盖实例凭证路径。 - `userKey` 是密钥，不应写入日志、状态响应或提交到版本库。 -- 容器访问宿主机 Gateway 时不能使用容器内的 `127.0.0.1`。应让 Gateway 监听 - `0.0.0.0`，并把 `baseUrl` 配成该 Docker 网络的宿主机网关地址。 -- 修改 memory 配置后需要重启 runtime，因为 Gateway 服务在 `EngineLoader` 启动时创建。 +- 修改共享 memory 配置后需要重启 runtime，因为 Gateway 相关对象在 `EngineLoader` 启动时装配。 diff --git a/app-instance/backend/beaver/engine/loader.py b/app-instance/backend/beaver/engine/loader.py index ad8d1d0..a523499 100644 --- a/app-instance/backend/beaver/engine/loader.py +++ b/app-instance/backend/beaver/engine/loader.py @@ -15,10 +15,16 @@ from beaver.engine.session import SessionManager from beaver.foundation.config import BeaverConfig, load_config from beaver.integrations.mcp import MCPConnectionManager from beaver.memory.curated.store import MemoryStore +from beaver.memory.gateway import ( + MemoryGatewayConfig, + MemoryGatewayCredentialStore, + MemoryGatewayService, + MemoryGatewayUserCredential, + default_memory_gateway_users_path, +) from beaver.memory.runs import RunMemoryStore from beaver.memory.skills import SkillLearningStore from beaver.services.memory_service import MemoryService -from beaver.services.memory_gateway_service import MemoryGatewayService from beaver.skills.drafts import DraftService from beaver.skills.learning import EvidenceSelector, SkillDraftSynthesizer, SkillLearningPipelineService, SkillLearningService from beaver.skills.learning.safety import SkillDraftSafetyChecker @@ -84,7 +90,9 @@ class EngineLoadResult: session_manager: SessionManager | None = None curated_memory_store: MemoryStore | None = None memory_service: MemoryService | None = None - memory_gateway_service: MemoryGatewayService | None = None + memory_gateway_config: MemoryGatewayConfig | None = None + memory_gateway_credentials: MemoryGatewayCredentialStore | None = None + memory_gateway_service_factory: Callable[[MemoryGatewayUserCredential], MemoryGatewayService] | None = None run_memory_store: RunMemoryStore | None = None skill_learning_store: SkillLearningStore | None = None tool_registry: ToolRegistry | None = None @@ -160,7 +168,8 @@ class EngineLoader: session_manager: SessionManager | None = None, curated_memory_store: MemoryStore | None = None, memory_service: MemoryService | None = None, - memory_gateway_service: MemoryGatewayService | None = None, + memory_gateway_credentials: MemoryGatewayCredentialStore | None = None, + memory_gateway_service_factory: Callable[[MemoryGatewayConfig, MemoryGatewayUserCredential], MemoryGatewayService] | None = None, run_memory_store: RunMemoryStore | None = None, skill_learning_store: SkillLearningStore | None = None, tool_registry: ToolRegistry | None = None, @@ -186,7 +195,8 @@ class EngineLoader: self._session_manager = session_manager self._curated_memory_store = curated_memory_store self._memory_service = memory_service - self._memory_gateway_service = memory_gateway_service + self._memory_gateway_credentials = memory_gateway_credentials + self._memory_gateway_service_factory = memory_gateway_service_factory self._run_memory_store = run_memory_store self._skill_learning_store = skill_learning_store self._tool_registry = tool_registry @@ -209,7 +219,11 @@ class EngineLoader: """装配当前主链需要的最小 runtime 对象。""" workspace = self.workspace - memory_gateway_service = self._resolve_memory_gateway_service() + ( + memory_gateway_config, + memory_gateway_credentials, + memory_gateway_service_factory, + ) = self._resolve_memory_gateway_components() session_manager = self._session_manager or SessionManager(workspace) curated_root = workspace / "memory" / "curated" @@ -306,12 +320,14 @@ class EngineLoader: config=self.config, tools=[spec.name for spec in tool_registry.list_specs()], skills=[record.name for record in skills_loader.list_skills(filter_unavailable=False)], - memory_stores=["curated", *(["memory_gateway"] if memory_gateway_service is not None else [])], + memory_stores=["curated", *(["memory_gateway"] if memory_gateway_service_factory is not None else [])], permissions=[], session_manager=session_manager, curated_memory_store=memory_service.get_store(), memory_service=memory_service, - memory_gateway_service=memory_gateway_service, + memory_gateway_config=memory_gateway_config, + memory_gateway_credentials=memory_gateway_credentials, + memory_gateway_service_factory=memory_gateway_service_factory, run_memory_store=run_memory_store, skill_learning_store=skill_learning_store, tool_registry=tool_registry, @@ -337,10 +353,16 @@ class EngineLoader: result.register_closeable("mcp_manager", lambda: _close_mcp_manager(mcp_manager)) return result - def _resolve_memory_gateway_service(self) -> MemoryGatewayService | None: + def _resolve_memory_gateway_components( + self, + ) -> tuple[ + MemoryGatewayConfig | None, + MemoryGatewayCredentialStore | None, + Callable[[MemoryGatewayUserCredential], MemoryGatewayService] | None, + ]: memory_config = self.config.memory if memory_config.mode == "curated": - return None + return None, None, None gateway_config = memory_config.gateway if memory_config.explicit and not gateway_config.is_configured: @@ -351,8 +373,18 @@ class EngineLoader: logger.warning( "Memory Gateway is not configured; continuing with curated memory only" ) - return None - return self._memory_gateway_service or MemoryGatewayService(gateway_config) + return None, None, None + + credential_store = self._memory_gateway_credentials or MemoryGatewayCredentialStore( + default_memory_gateway_users_path() + ) + + def factory(credential: MemoryGatewayUserCredential) -> MemoryGatewayService: + if self._memory_gateway_service_factory is not None: + return self._memory_gateway_service_factory(gateway_config, credential) + return MemoryGatewayService(gateway_config, credential) + + return gateway_config, credential_store, factory def _close_mcp_manager(manager: MCPConnectionManager) -> None: diff --git a/app-instance/backend/beaver/engine/loop.py b/app-instance/backend/beaver/engine/loop.py index 3df3160..3798cef 100644 --- a/app-instance/backend/beaver/engine/loop.py +++ b/app-instance/backend/beaver/engine/loop.py @@ -227,6 +227,7 @@ class AgentLoop: session_id: str | None = None, source: str = "direct", user_id: str | None = None, + gateway_user_id: str | None = None, title: str | None = None, execution_context: str | None = None, skill_selection_context: str | None = None, @@ -279,6 +280,7 @@ class AgentLoop: session_id=session_id, source=source, user_id=user_id, + gateway_user_id=gateway_user_id, title=title, execution_context=execution_context, skill_selection_context=skill_selection_context, @@ -319,6 +321,7 @@ class AgentLoop: session_id: str | None = None, source: str = "direct", user_id: str | None = None, + gateway_user_id: str | None = None, title: str | None = None, execution_context: str | None = None, skill_selection_context: str | None = None, @@ -360,6 +363,13 @@ class AgentLoop: """ loaded = self.boot() + memory_gateway_service = None + gateway_credential_store = getattr(loaded, "memory_gateway_credentials", None) + gateway_service_factory = getattr(loaded, "memory_gateway_service_factory", None) + if gateway_user_id and gateway_credential_store is not None and gateway_service_factory is not None: + gateway_credential = gateway_credential_store.get(gateway_user_id) + if gateway_credential is not None: + memory_gateway_service = gateway_service_factory(gateway_credential) session_manager = self._require_loaded("session_manager") memory_service = self._require_loaded("memory_service") context_builder = self._require_loaded("context_builder") @@ -482,7 +492,6 @@ class AgentLoop: final_model: str | None = resolved_model run_started_at = self._utc_now() activated_receipts: list[SkillActivationReceipt] = [] - memory_gateway_service = getattr(loaded, "memory_gateway_service", None) try: bundle = provider_bundle or make_provider_bundle( model=resolved_model, diff --git a/app-instance/backend/beaver/foundation/config/__init__.py b/app-instance/backend/beaver/foundation/config/__init__.py index 57b183c..bc95c9c 100644 --- a/app-instance/backend/beaver/foundation/config/__init__.py +++ b/app-instance/backend/beaver/foundation/config/__init__.py @@ -1,6 +1,6 @@ """Configuration models and loaders.""" -from .loader import default_config_path, load_config +from .loader import default_config_path, default_memory_config_path, load_config from .schema import ( AgentDefaultsConfig, AuthzConfig, @@ -26,5 +26,6 @@ __all__ = [ "ProviderConfig", "ToolsConfig", "default_config_path", + "default_memory_config_path", "load_config", ] diff --git a/app-instance/backend/beaver/foundation/config/loader.py b/app-instance/backend/beaver/foundation/config/loader.py index 7d04389..1e845ce 100644 --- a/app-instance/backend/beaver/foundation/config/loader.py +++ b/app-instance/backend/beaver/foundation/config/loader.py @@ -55,6 +55,16 @@ def default_config_path(*, workspace: str | Path | None = None) -> Path: return root / ".beaver" / "config.json" +def default_memory_config_path() -> Path: + """Resolve the shared Memory Gateway config path.""" + + explicit = os.getenv("BEAVER_MEMORY_CONFIG_PATH") + if explicit: + return Path(explicit).expanduser() + + return Path(__file__).resolve().parents[3] / "memory" / "config.json" + + def load_config( *, workspace: str | Path | None = None, @@ -63,24 +73,38 @@ def load_config( """Load backend config; missing config is treated as an empty config.""" path = Path(config_path).expanduser() if config_path is not None else default_config_path(workspace=workspace) + data: dict[str, Any] | None = None + if path.exists(): + loaded = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(loaded, dict): + raise ValueError(f"Beaver config must be a JSON object: {path}") + data = loaded + memory_data = _load_memory_config_data() + + return BeaverConfig( + agents_defaults=_parse_agent_defaults(data or {}), + providers=_parse_providers((data or {}).get("providers")), + embedding=_parse_embedding(data or {}), + tools=_parse_tools((data or {}).get("tools")) if data is not None else ToolsConfig(), + authz=_parse_authz((data or {}).get("authz")), + channels=_parse_channels((data or {}).get("channels")), + backend_identity=_parse_backend_identity( + (data or {}).get("backend_identity") or (data or {}).get("backendIdentity") + ), + memory=_parse_memory(memory_data), + config_path=path, + ) + + +def _load_memory_config_data() -> dict[str, Any]: + path = default_memory_config_path() if not path.exists(): - return BeaverConfig(config_path=path) + return {} data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict): - raise ValueError(f"Beaver config must be a JSON object: {path}") - - return BeaverConfig( - agents_defaults=_parse_agent_defaults(data), - providers=_parse_providers(data.get("providers")), - embedding=_parse_embedding(data), - tools=_parse_tools(data.get("tools")), - authz=_parse_authz(data.get("authz")), - channels=_parse_channels(data.get("channels")), - backend_identity=_parse_backend_identity(data.get("backend_identity") or data.get("backendIdentity")), - memory=_parse_memory(data), - config_path=path, - ) + raise ValueError(f"Beaver memory config must be a JSON object: {path}") + return data def _parse_agent_defaults(data: dict[str, Any]) -> AgentDefaultsConfig: @@ -269,12 +293,10 @@ def _parse_memory(data: dict[str, Any]) -> MemoryConfig: scope = ( _string_list(gateway_raw.get("scope")) if "scope" in gateway_raw - else ["current_chat", "resources"] + else MemoryGatewayConfig().scope ) gateway = MemoryGatewayConfig( base_url=_string(gateway_raw.get("baseUrl") or gateway_raw.get("base_url")) or "", - user_id=_string(gateway_raw.get("userId") or gateway_raw.get("user_id")) or "", - user_key=_string(gateway_raw.get("userKey") or gateway_raw.get("user_key")) or "", app_id=_string(gateway_raw.get("appId") or gateway_raw.get("app_id")) or "default", project_id=_string(gateway_raw.get("projectId") or gateway_raw.get("project_id")) or "default", scope=scope, @@ -283,15 +305,8 @@ def _parse_memory(data: dict[str, Any]) -> MemoryConfig: ) if mode == "hybrid" and explicit: - missing: list[str] = [] if not gateway.base_url: - missing.append("baseUrl") - if not gateway.user_id: - missing.append("userId") - if not gateway.user_key: - missing.append("userKey") - if missing: - raise ValueError(f"Explicit hybrid memory requires gateway fields: {', '.join(missing)}") + raise ValueError("Explicit hybrid memory requires gateway.baseUrl") allowed_scopes = {"current_chat", "resources", "all_user_memory"} if not gateway.scope or any(scope not in allowed_scopes for scope in gateway.scope): raise ValueError("memory.gateway.scope contains an unsupported value") diff --git a/app-instance/backend/beaver/foundation/config/schema.py b/app-instance/backend/beaver/foundation/config/schema.py index 0c39a4f..3d3cd30 100644 --- a/app-instance/backend/beaver/foundation/config/schema.py +++ b/app-instance/backend/beaver/foundation/config/schema.py @@ -6,6 +6,8 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any +from beaver.memory.gateway import MemoryConfig, MemoryGatewayConfig + @dataclass(slots=True) class ProviderConfig: @@ -115,33 +117,6 @@ class BackendIdentityConfig: public_base_url: str = "" -@dataclass(slots=True) -class MemoryGatewayConfig: - """Fixed Memory Gateway settings for one Beaver instance.""" - - base_url: str = "" - user_id: str = "" - user_key: str = field(default="", repr=False) - app_id: str = "default" - project_id: str = "default" - scope: list[str] = field(default_factory=lambda: ["current_chat", "resources"]) - top_k: int = 8 - timeout_seconds: float = 10.0 - - @property - def is_configured(self) -> bool: - return bool(_clean(self.base_url) and _clean(self.user_id) and _clean(self.user_key)) - - -@dataclass(slots=True) -class MemoryConfig: - """Curated baseline plus optional Memory Gateway layer.""" - - mode: str = "hybrid" - explicit: bool = False - gateway: MemoryGatewayConfig = field(default_factory=MemoryGatewayConfig) - - @dataclass(slots=True) class BeaverConfig: """Config loaded once per backend sandbox instance.""" diff --git a/app-instance/backend/beaver/integrations/memory_gateway/__init__.py b/app-instance/backend/beaver/integrations/memory_gateway/__init__.py deleted file mode 100644 index 2aaab3a..0000000 --- a/app-instance/backend/beaver/integrations/memory_gateway/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Memory Gateway HTTP integration.""" - -from .client import MemoryGatewayClient, MemoryGatewayClientError - -__all__ = ["MemoryGatewayClient", "MemoryGatewayClientError"] diff --git a/app-instance/backend/beaver/interfaces/web/app.py b/app-instance/backend/beaver/interfaces/web/app.py index 945ffda..90cc15e 100644 --- a/app-instance/backend/beaver/interfaces/web/app.py +++ b/app-instance/backend/beaver/interfaces/web/app.py @@ -5,6 +5,7 @@ from __future__ import annotations import json import asyncio import io +import logging import mimetypes import os import re @@ -21,6 +22,13 @@ from typing import Any from beaver.engine.providers.registry import PROVIDERS, find_by_name from beaver.foundation.config import default_config_path, load_config from beaver.foundation.events import ChannelIdentity, InboundMessage +from beaver.memory.gateway import ( + MemoryGatewayClient, + MemoryGatewayClientError, + MemoryGatewayCredentialStore, + MemoryGatewayUserCredential, + default_memory_gateway_users_path, +) from beaver.interfaces.channels.runtime import ChannelRuntime from beaver.interfaces.channels.connections import ( ChannelConnectionStore, @@ -97,6 +105,8 @@ from .schemas import ( WebStatusResponse, ) +logger = logging.getLogger(__name__) + try: from fastapi import FastAPI, File, Form, Header, HTTPException, Request, UploadFile, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware @@ -588,6 +598,10 @@ def create_app( app.state.auth_tokens = {} app.state.handoff_codes = {} app.state.auth_file = Path(os.getenv("BEAVER_AUTH_FILE") or "") + app.state.memory_gateway_credential_store = MemoryGatewayCredentialStore( + default_memory_gateway_users_path() + ) + app.state.memory_gateway_client_factory = lambda config: MemoryGatewayClient(config) max_file_size = 50 * 1024 * 1024 max_user_file_upload_size = _int_env("BEAVER_USER_FILES_MAX_UPLOAD_BYTES", 5 * 1024 * 1024 * 1024) user_file_upload_part_size = _int_env("BEAVER_USER_FILES_UPLOAD_PART_SIZE", 10 * 1024 * 1024) @@ -1103,6 +1117,30 @@ def create_app( users[username] = password _save_auth_users(auth_file, users) + if config.memory.mode == "hybrid" and config.memory.gateway.is_configured: + try: + gateway_client = app.state.memory_gateway_client_factory(config.memory.gateway) + gateway_payload = await gateway_client.create_user(username) + gateway_user_id = _clean_text(gateway_payload.get("user_id")) + gateway_user_key = _clean_text(gateway_payload.get("user_key")) + if not gateway_user_id or not gateway_user_key: + raise MemoryGatewayClientError("create_user", "invalid_response") + app.state.memory_gateway_credential_store.save( + username, + MemoryGatewayUserCredential( + user_id=gateway_user_id, + user_key=gateway_user_key, + ), + ) + except MemoryGatewayClientError as exc: + logger.warning( + "Memory Gateway user provisioning failed for Beaver user %s: operation=%s category=%s status_code=%s", + username, + exc.operation, + exc.category, + exc.status_code, + ) + token = _issue_web_token(app, username) handoff_code, handoff_expires_at = _issue_handoff_code(app, username, token) backend_connection = { @@ -2445,7 +2483,11 @@ def create_app( 503: {"model": WebErrorResponse}, }, ) - async def chat(request: Request, payload: WebChatRequest) -> WebChatResponse: + async def chat( + request: Request, + payload: WebChatRequest, + authorization: str | None = Header(default=None), + ) -> WebChatResponse: agent_service = get_agent_service(request) message = payload.message.strip() if not message: @@ -2496,10 +2538,12 @@ def create_app( embedding_target = _model_dump(payload.embedding_target) try: + gateway_user_id = _optional_web_user(app, authorization) direct_kwargs = { "session_id": payload.session_id, "source": "web", "user_id": payload.user_id, + "gateway_user_id": gateway_user_id, "title": payload.title, "execution_context": payload.execution_context, "prompt_locale": payload.prompt_locale, @@ -2558,6 +2602,7 @@ def create_app( await websocket.send_json({"type": "error", "error": "AgentService is not ready"}) await websocket.close(code=1011) return + gateway_user_id = _web_user_from_token(app, websocket.query_params.get("token")) while True: try: @@ -2616,6 +2661,7 @@ def create_app( "session_id": session_id, "source": "websocket", "user_id": _clean_text(payload.get("user_id")) or None, + "gateway_user_id": gateway_user_id, "title": _clean_text(payload.get("title")) or None, "execution_context": _clean_text(payload.get("execution_context")) or None, "prompt_locale": _clean_text(payload.get("prompt_locale")) or None, @@ -3680,6 +3726,22 @@ def _require_web_user(app: FastAPI, authorization: str | None) -> str: return username +def _optional_web_user(app: FastAPI, authorization: str | None) -> str | None: + if not authorization: + return None + prefix = "bearer " + if not authorization.lower().startswith(prefix): + return None + return _web_user_from_token(app, authorization[len(prefix):].strip()) + + +def _web_user_from_token(app: FastAPI, token: str | None) -> str | None: + cleaned = _clean_text(token) + if not cleaned: + return None + return app.state.auth_tokens.get(cleaned) + + def _backend_connection_view(request: Request) -> dict[str, Any]: public_base_url = ( os.getenv("BEAVER_BACKEND_IDENTITY__PUBLIC_BASE_URL") diff --git a/app-instance/backend/beaver/memory/gateway/__init__.py b/app-instance/backend/beaver/memory/gateway/__init__.py new file mode 100644 index 0000000..563914d --- /dev/null +++ b/app-instance/backend/beaver/memory/gateway/__init__.py @@ -0,0 +1,23 @@ +"""Memory Gateway support.""" + +from .client import MemoryGatewayClient, MemoryGatewayClientError +from .config import MemoryConfig, MemoryGatewayConfig +from .credentials import ( + MemoryGatewayCredentialStore, + MemoryGatewayUserCredential, + default_memory_gateway_users_path, +) +from .service import GatewayPersistOutcome, GatewayRecallOutcome, MemoryGatewayService + +__all__ = [ + "GatewayPersistOutcome", + "GatewayRecallOutcome", + "MemoryConfig", + "MemoryGatewayCredentialStore", + "MemoryGatewayClient", + "MemoryGatewayClientError", + "MemoryGatewayConfig", + "MemoryGatewayService", + "MemoryGatewayUserCredential", + "default_memory_gateway_users_path", +] diff --git a/app-instance/backend/beaver/integrations/memory_gateway/client.py b/app-instance/backend/beaver/memory/gateway/client.py similarity index 89% rename from app-instance/backend/beaver/integrations/memory_gateway/client.py rename to app-instance/backend/beaver/memory/gateway/client.py index a6fbe52..c82dae6 100644 --- a/app-instance/backend/beaver/integrations/memory_gateway/client.py +++ b/app-instance/backend/beaver/memory/gateway/client.py @@ -6,7 +6,7 @@ from typing import Any import httpx -from beaver.foundation.config import MemoryGatewayConfig +from .config import MemoryGatewayConfig class MemoryGatewayClientError(RuntimeError): @@ -21,7 +21,7 @@ class MemoryGatewayClientError(RuntimeError): class MemoryGatewayClient: - """HTTP transport for search, add, and flush operations.""" + """HTTP transport for search, add, flush, and provisioning operations.""" def __init__( self, @@ -32,6 +32,9 @@ class MemoryGatewayClient: self.config = config self.transport = transport + async def create_user(self, user_id: str) -> dict[str, Any]: + return await self._post("create_user", "/users", {"user_id": user_id}) + async def search(self, payload: dict[str, Any]) -> dict[str, Any]: return await self._post("search", "/memories/search", payload) diff --git a/app-instance/backend/beaver/memory/gateway/config.py b/app-instance/backend/beaver/memory/gateway/config.py new file mode 100644 index 0000000..9406e6e --- /dev/null +++ b/app-instance/backend/beaver/memory/gateway/config.py @@ -0,0 +1,32 @@ +"""Configuration models for the Memory Gateway layer.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(slots=True) +class MemoryGatewayConfig: + """Shared non-secret Memory Gateway settings.""" + + base_url: str = "" + app_id: str = "default" + project_id: str = "default" + scope: list[str] = field( + default_factory=lambda: ["current_chat", "resources", "all_user_memory"] + ) + top_k: int = 8 + timeout_seconds: float = 10.0 + + @property + def is_configured(self) -> bool: + return bool(self.base_url.strip()) + + +@dataclass(slots=True) +class MemoryConfig: + """Curated baseline plus optional Memory Gateway layer.""" + + mode: str = "hybrid" + explicit: bool = False + gateway: MemoryGatewayConfig = field(default_factory=MemoryGatewayConfig) diff --git a/app-instance/backend/beaver/memory/gateway/credentials.py b/app-instance/backend/beaver/memory/gateway/credentials.py new file mode 100644 index 0000000..333556b --- /dev/null +++ b/app-instance/backend/beaver/memory/gateway/credentials.py @@ -0,0 +1,75 @@ +"""Per-instance credential storage for Memory Gateway users.""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass(slots=True) +class MemoryGatewayUserCredential: + user_id: str + user_key: str = field(repr=False) + + +class MemoryGatewayCredentialStore: + """Persist Beaver username -> Gateway credential mappings.""" + + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + + def get(self, username: str) -> MemoryGatewayUserCredential | None: + users = self._load_users() + payload = users.get(username) + if not isinstance(payload, dict): + return None + user_id = str(payload.get("userId") or "").strip() + user_key = str(payload.get("userKey") or "").strip() + if not user_id or not user_key: + return None + return MemoryGatewayUserCredential(user_id=user_id, user_key=user_key) + + def save(self, username: str, credential: MemoryGatewayUserCredential) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + users = self._load_users() + users[username] = { + "userId": credential.user_id, + "userKey": credential.user_key, + } + payload = {"users": dict(sorted(users.items()))} + fd, tmp_name = tempfile.mkstemp( + prefix=f".{self.path.name}.", + suffix=".tmp", + dir=str(self.path.parent), + ) + tmp_path = Path(tmp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2) + handle.write("\n") + os.chmod(tmp_path, 0o600) + os.replace(tmp_path, self.path) + os.chmod(self.path, 0o600) + finally: + if tmp_path.exists(): + tmp_path.unlink() + + def _load_users(self) -> dict[str, Any]: + if not self.path.exists(): + return {} + data = json.loads(self.path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return {} + users = data.get("users") + return users if isinstance(users, dict) else {} + + +def default_memory_gateway_users_path() -> Path: + raw = os.getenv("BEAVER_MEMORY_GATEWAY_USERS_PATH") + if raw: + return Path(raw) + return Path.home() / ".beaver" / "memory_gateway_users.json" diff --git a/app-instance/backend/beaver/services/memory_gateway_service.py b/app-instance/backend/beaver/memory/gateway/service.py similarity index 88% rename from app-instance/backend/beaver/services/memory_gateway_service.py rename to app-instance/backend/beaver/memory/gateway/service.py index 1616d00..7018faa 100644 --- a/app-instance/backend/beaver/services/memory_gateway_service.py +++ b/app-instance/backend/beaver/memory/gateway/service.py @@ -6,8 +6,9 @@ import json from dataclasses import dataclass, field from typing import Any -from beaver.foundation.config import MemoryGatewayConfig -from beaver.integrations.memory_gateway import MemoryGatewayClient, MemoryGatewayClientError +from .client import MemoryGatewayClient, MemoryGatewayClientError +from .config import MemoryGatewayConfig +from .credentials import MemoryGatewayUserCredential _RECALL_FIELDS = ("id", "session_id", "text", "score", "source_scope", "resource_uri") @@ -33,16 +34,18 @@ class MemoryGatewayService: def __init__( self, config: MemoryGatewayConfig, + credential: MemoryGatewayUserCredential, *, client: MemoryGatewayClient | None = None, ) -> None: self.config = config + self.credential = credential self.client = client or MemoryGatewayClient(config) async def recall_before_run(self, *, session_id: str, query: str) -> GatewayRecallOutcome: payload = { - "user_id": self.config.user_id, - "user_key": self.config.user_key, + "user_id": self.credential.user_id, + "user_key": self.credential.user_key, "conversation_id": session_id, "query": query, "scope": list(self.config.scope), @@ -90,8 +93,8 @@ class MemoryGatewayService: ) -> GatewayPersistOutcome: gateway_session_id = f"chat:{session_id}" common = { - "user_id": self.config.user_id, - "user_key": self.config.user_key, + "user_id": self.credential.user_id, + "user_key": self.credential.user_key, "session_id": gateway_session_id, "app_id": self.config.app_id, "project_id": self.config.project_id, @@ -100,7 +103,7 @@ class MemoryGatewayService: **common, "messages": [ { - "sender_id": self.config.user_id, + "sender_id": self.credential.user_id, "role": "user", "timestamp": user_timestamp_ms, "content": user_text, diff --git a/app-instance/backend/beaver/services/__init__.py b/app-instance/backend/beaver/services/__init__.py index 4830808..226917d 100644 --- a/app-instance/backend/beaver/services/__init__.py +++ b/app-instance/backend/beaver/services/__init__.py @@ -1,6 +1,6 @@ """Application services for Beaver.""" -__all__ = ["AgentService", "CronService", "MemoryGatewayService", "MemoryService"] +__all__ = ["AgentService", "CronService", "MemoryService"] def __getattr__(name: str): @@ -12,10 +12,6 @@ def __getattr__(name: str): from .memory_service import MemoryService return MemoryService - if name == "MemoryGatewayService": - from .memory_gateway_service import MemoryGatewayService - - return MemoryGatewayService if name == "CronService": from .cron_service import CronService diff --git a/app-instance/backend/memory/config.json b/app-instance/backend/memory/config.json new file mode 100644 index 0000000..6947395 --- /dev/null +++ b/app-instance/backend/memory/config.json @@ -0,0 +1,13 @@ +{ + "memory": { + "mode": "hybrid", + "gateway": { + "baseUrl": "http://10.6.80.123:8010", + "appId": "default", + "projectId": "default", + "scope": ["current_chat", "resources", "all_user_memory"], + "topK": 8, + "timeoutSeconds": 10 + } + } +} diff --git a/app-instance/backend/tests/unit/test_config_loader.py b/app-instance/backend/tests/unit/test_config_loader.py index a175e9b..e43c9a7 100644 --- a/app-instance/backend/tests/unit/test_config_loader.py +++ b/app-instance/backend/tests/unit/test_config_loader.py @@ -12,6 +12,39 @@ from beaver.interfaces.web.app import create_app, _reload_agent_config from beaver.services.agent_service import AgentService +def test_load_config_reads_shared_memory_config(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({}), encoding="utf-8") + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text( + json.dumps( + { + "memory": { + "mode": "hybrid", + "gateway": { + "baseUrl": "http://172.19.207.37:8010", + "appId": "default", + "projectId": "default", + "scope": ["current_chat", "resources", "all_user_memory"], + "topK": 8, + "timeoutSeconds": 10, + }, + } + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(memory_config_path)) + + config = load_config(config_path=config_path) + + assert config.memory.mode == "hybrid" + assert config.memory.gateway.base_url == "http://172.19.207.37:8010" + assert config.memory.gateway.scope == ["current_chat", "resources", "all_user_memory"] + assert config.memory.gateway.top_k == 8 + assert config.memory.gateway.timeout_seconds == 10 + + def test_load_config_reads_current_instance_shape(tmp_path) -> None: config_path = tmp_path / "config.json" config_path.write_text( @@ -477,17 +510,25 @@ def test_load_config_adds_managed_local_mcp_servers(tmp_path) -> None: assert "beaver.interfaces.mcp.tools_server" in local.args -def test_missing_memory_config_defaults_to_implicit_hybrid(tmp_path) -> None: +def test_missing_memory_config_defaults_to_implicit_hybrid( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(tmp_path / "missing-memory.json")) config = load_config(config_path=tmp_path / "missing.json") assert config.memory.mode == "hybrid" assert config.memory.explicit is False - assert config.memory.gateway.scope == ["current_chat", "resources"] + assert config.memory.gateway.scope == ["current_chat", "resources", "all_user_memory"] -def test_load_config_reads_explicit_curated_memory_mode(tmp_path) -> None: +def test_load_config_reads_explicit_curated_memory_mode( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: config_path = tmp_path / "config.json" - config_path.write_text(json.dumps({"memory": {"mode": "curated"}}), encoding="utf-8") + config_path.write_text(json.dumps({}), encoding="utf-8") + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text(json.dumps({"memory": {"mode": "curated"}}), encoding="utf-8") + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(memory_config_path)) config = load_config(config_path=config_path) @@ -495,17 +536,19 @@ def test_load_config_reads_explicit_curated_memory_mode(tmp_path) -> None: assert config.memory.explicit is True -def test_load_config_reads_explicit_hybrid_gateway_settings(tmp_path) -> None: +def test_load_config_reads_explicit_hybrid_gateway_settings( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: config_path = tmp_path / "config.json" - config_path.write_text( + config_path.write_text(json.dumps({}), encoding="utf-8") + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text( json.dumps( { "memory": { "mode": "hybrid", "gateway": { "baseUrl": "http://127.0.0.1:8010", - "userId": "gateway-user", - "userKey": "uk_secret", "appId": "beaver", "projectId": "sandbox", "scope": ["current_chat", "resources"], @@ -517,14 +560,13 @@ def test_load_config_reads_explicit_hybrid_gateway_settings(tmp_path) -> None: ), encoding="utf-8", ) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(memory_config_path)) config = load_config(config_path=config_path) assert config.memory.mode == "hybrid" assert config.memory.explicit is True assert config.memory.gateway.base_url == "http://127.0.0.1:8010" - assert config.memory.gateway.user_id == "gateway-user" - assert config.memory.gateway.user_key == "uk_secret" assert config.memory.gateway.app_id == "beaver" assert config.memory.gateway.project_id == "sandbox" assert config.memory.gateway.scope == ["current_chat", "resources"] @@ -532,41 +574,33 @@ def test_load_config_reads_explicit_hybrid_gateway_settings(tmp_path) -> None: assert config.memory.gateway.timeout_seconds == 12.5 -def test_explicit_hybrid_requires_gateway_credentials_without_leaking_secret(tmp_path) -> None: +def test_explicit_hybrid_requires_gateway_base_url(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: config_path = tmp_path / "config.json" - config_path.write_text( - json.dumps( - { - "memory": { - "mode": "hybrid", - "gateway": { - "baseUrl": "http://127.0.0.1:8010", - "userKey": "uk_super_secret", - }, - } - } - ), + config_path.write_text(json.dumps({}), encoding="utf-8") + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text( + json.dumps({"memory": {"mode": "hybrid", "gateway": {"appId": "beaver"}}}), encoding="utf-8", ) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(memory_config_path)) with pytest.raises(ValueError) as exc_info: load_config(config_path=config_path) - assert "userId" in str(exc_info.value) - assert "uk_super_secret" not in str(exc_info.value) + assert "baseUrl" in str(exc_info.value) -def test_hybrid_memory_rejects_unknown_scope(tmp_path) -> None: +def test_hybrid_memory_rejects_unknown_scope(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: config_path = tmp_path / "config.json" - config_path.write_text( + config_path.write_text(json.dumps({}), encoding="utf-8") + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text( json.dumps( { "memory": { "mode": "hybrid", "gateway": { "baseUrl": "http://127.0.0.1:8010", - "userId": "gateway-user", - "userKey": "uk_secret", "scope": ["current_chat", "unknown"], }, } @@ -574,22 +608,23 @@ def test_hybrid_memory_rejects_unknown_scope(tmp_path) -> None: ), encoding="utf-8", ) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(memory_config_path)) with pytest.raises(ValueError, match="scope"): load_config(config_path=config_path) -def test_hybrid_memory_rejects_empty_scope(tmp_path) -> None: +def test_hybrid_memory_rejects_empty_scope(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: config_path = tmp_path / "config.json" - config_path.write_text( + config_path.write_text(json.dumps({}), encoding="utf-8") + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text( json.dumps( { "memory": { "mode": "hybrid", "gateway": { "baseUrl": "http://127.0.0.1:8010", - "userId": "gateway-user", - "userKey": "uk_secret", "scope": [], }, } @@ -597,6 +632,7 @@ def test_hybrid_memory_rejects_empty_scope(tmp_path) -> None: ), encoding="utf-8", ) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(memory_config_path)) with pytest.raises(ValueError, match="scope"): load_config(config_path=config_path) @@ -610,18 +646,21 @@ def test_hybrid_memory_rejects_empty_scope(tmp_path) -> None: ({"timeoutSeconds": 0}, "timeoutSeconds"), ], ) -def test_hybrid_memory_rejects_invalid_limits(tmp_path, gateway_override, expected_error) -> None: +def test_hybrid_memory_rejects_invalid_limits( + tmp_path, gateway_override, expected_error, monkeypatch: pytest.MonkeyPatch +) -> None: config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({}), encoding="utf-8") gateway = { "baseUrl": "http://127.0.0.1:8010", - "userId": "gateway-user", - "userKey": "uk_secret", **gateway_override, } - config_path.write_text( + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text( json.dumps({"memory": {"mode": "hybrid", "gateway": gateway}}), encoding="utf-8", ) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(memory_config_path)) with pytest.raises(ValueError, match=expected_error): load_config(config_path=config_path) diff --git a/app-instance/backend/tests/unit/test_memory_gateway_agent_loop.py b/app-instance/backend/tests/unit/test_memory_gateway_agent_loop.py index 145dad1..8b0ff9d 100644 --- a/app-instance/backend/tests/unit/test_memory_gateway_agent_loop.py +++ b/app-instance/backend/tests/unit/test_memory_gateway_agent_loop.py @@ -8,8 +8,13 @@ from beaver.engine import AgentLoop, EngineLoader from beaver.engine.providers.base import LLMProvider, LLMResponse from beaver.engine.providers.factory import ProviderBundle from beaver.foundation.config import BeaverConfig, MemoryConfig, MemoryGatewayConfig -from beaver.integrations.memory_gateway import MemoryGatewayClientError -from beaver.services.memory_gateway_service import GatewayPersistOutcome, GatewayRecallOutcome +from beaver.memory.gateway import ( + GatewayPersistOutcome, + GatewayRecallOutcome, + MemoryGatewayClientError, + MemoryGatewayCredentialStore, + MemoryGatewayUserCredential, +) class RecordingProvider(LLMProvider): @@ -74,8 +79,6 @@ def _hybrid_config() -> BeaverConfig: explicit=True, gateway=MemoryGatewayConfig( base_url="http://gateway.test", - user_id="gateway-user", - user_key="uk_secret", scope=["current_chat", "resources"], ), ) @@ -93,11 +96,24 @@ def _write_curated_user_memory(workspace: Path) -> None: (root / "USER.md").write_text("The user prefers concise answers.", encoding="utf-8") -def _run(loop: AgentLoop, provider: LLMProvider, *, session_id: str = "web:gateway-test"): +def _gateway_store(tmp_path: Path) -> MemoryGatewayCredentialStore: + store = MemoryGatewayCredentialStore(tmp_path / "memory_gateway_users.json") + store.save("tom", MemoryGatewayUserCredential(user_id="gateway-user", user_key="uk_secret")) + return store + + +def _run( + loop: AgentLoop, + provider: LLMProvider, + *, + session_id: str = "web:gateway-test", + gateway_user_id: str | None = "tom", +): return asyncio.run( loop.process_direct( "What should I remember?", session_id=session_id, + gateway_user_id=gateway_user_id, provider_bundle=_bundle(provider), include_skill_assembly=False, include_tools=False, @@ -134,7 +150,8 @@ def test_hybrid_run_keeps_curated_context_and_persists_gateway_turn(tmp_path: Pa loader=EngineLoader( workspace=tmp_path, config=_hybrid_config(), - memory_gateway_service=gateway, + memory_gateway_credentials=_gateway_store(tmp_path), + memory_gateway_service_factory=lambda _config, _credential: gateway, ) ) @@ -182,7 +199,8 @@ def test_gateway_recall_failure_is_audited_without_changing_result(tmp_path: Pat loader=EngineLoader( workspace=tmp_path, config=_hybrid_config(), - memory_gateway_service=gateway, + memory_gateway_credentials=_gateway_store(tmp_path), + memory_gateway_service_factory=lambda _config, _credential: gateway, ) ) @@ -210,7 +228,8 @@ def test_gateway_add_failure_skips_flush_audit_and_preserves_result(tmp_path: Pa loader=EngineLoader( workspace=tmp_path, config=_hybrid_config(), - memory_gateway_service=gateway, + memory_gateway_credentials=_gateway_store(tmp_path), + memory_gateway_service_factory=lambda _config, _credential: gateway, ) ) @@ -235,7 +254,8 @@ def test_gateway_flush_failure_records_add_success_and_flush_failure(tmp_path: P loader=EngineLoader( workspace=tmp_path, config=_hybrid_config(), - memory_gateway_service=gateway, + memory_gateway_credentials=_gateway_store(tmp_path), + memory_gateway_service_factory=lambda _config, _credential: gateway, ) ) @@ -276,7 +296,8 @@ def test_failed_run_is_not_persisted_to_gateway(tmp_path: Path) -> None: loader=EngineLoader( workspace=tmp_path, config=_hybrid_config(), - memory_gateway_service=gateway, + memory_gateway_credentials=_gateway_store(tmp_path), + memory_gateway_service_factory=lambda _config, _credential: gateway, ) ) @@ -286,3 +307,23 @@ def test_failed_run_is_not_persisted_to_gateway(tmp_path: Path) -> None: assert gateway.recall_calls assert gateway.persist_calls == [] loop.close() + + +def test_missing_gateway_identity_skips_gateway_calls(tmp_path: Path) -> None: + gateway = FakeGatewayService() + provider = RecordingProvider(LLMResponse(content="Curated only.", finish_reason="stop")) + loop = AgentLoop( + loader=EngineLoader( + workspace=tmp_path, + config=_hybrid_config(), + memory_gateway_credentials=_gateway_store(tmp_path), + memory_gateway_service_factory=lambda _config, _credential: gateway, + ) + ) + + result = _run(loop, provider, session_id="web:no-gateway-user", gateway_user_id=None) + + assert result.output_text == "Curated only." + assert gateway.recall_calls == [] + assert gateway.persist_calls == [] + loop.close() diff --git a/app-instance/backend/tests/unit/test_memory_gateway_credentials.py b/app-instance/backend/tests/unit/test_memory_gateway_credentials.py new file mode 100644 index 0000000..cd7f8e3 --- /dev/null +++ b/app-instance/backend/tests/unit/test_memory_gateway_credentials.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import json +import stat + +from beaver.memory.gateway import ( + MemoryGatewayCredentialStore, + MemoryGatewayUserCredential, +) + + +def test_credential_store_returns_none_for_missing_user(tmp_path) -> None: + store = MemoryGatewayCredentialStore(tmp_path / "memory_gateway_users.json") + + assert store.get("tom") is None + + +def test_credential_store_round_trips_multiple_users(tmp_path) -> None: + path = tmp_path / "memory_gateway_users.json" + store = MemoryGatewayCredentialStore(path) + + store.save("tom", MemoryGatewayUserCredential(user_id="tom", user_key="uk_tom")) + store.save("alice", MemoryGatewayUserCredential(user_id="alice", user_key="uk_alice")) + + assert store.get("tom") == MemoryGatewayUserCredential(user_id="tom", user_key="uk_tom") + assert store.get("alice") == MemoryGatewayUserCredential(user_id="alice", user_key="uk_alice") + + payload = json.loads(path.read_text(encoding="utf-8")) + assert payload == { + "users": { + "alice": {"userId": "alice", "userKey": "uk_alice"}, + "tom": {"userId": "tom", "userKey": "uk_tom"}, + } + } + + +def test_credential_store_update_preserves_other_users(tmp_path) -> None: + path = tmp_path / "memory_gateway_users.json" + store = MemoryGatewayCredentialStore(path) + store.save("tom", MemoryGatewayUserCredential(user_id="tom", user_key="uk_old")) + store.save("alice", MemoryGatewayUserCredential(user_id="alice", user_key="uk_alice")) + + store.save("tom", MemoryGatewayUserCredential(user_id="tom", user_key="uk_new")) + + assert store.get("tom") == MemoryGatewayUserCredential(user_id="tom", user_key="uk_new") + assert store.get("alice") == MemoryGatewayUserCredential(user_id="alice", user_key="uk_alice") + + +def test_credential_store_masks_secret_in_repr_and_uses_private_mode(tmp_path) -> None: + path = tmp_path / "memory_gateway_users.json" + credential = MemoryGatewayUserCredential(user_id="tom", user_key="uk_super_secret") + store = MemoryGatewayCredentialStore(path) + + store.save("tom", credential) + + assert "uk_super_secret" not in repr(credential) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert not any(child.suffix == ".tmp" for child in tmp_path.iterdir()) diff --git a/app-instance/backend/tests/unit/test_memory_gateway_loader.py b/app-instance/backend/tests/unit/test_memory_gateway_loader.py index e3be31b..6e757c9 100644 --- a/app-instance/backend/tests/unit/test_memory_gateway_loader.py +++ b/app-instance/backend/tests/unit/test_memory_gateway_loader.py @@ -6,6 +6,7 @@ import pytest from beaver.engine import EngineLoader from beaver.foundation.config import BeaverConfig, MemoryConfig, MemoryGatewayConfig +from beaver.memory.gateway import MemoryGatewayCredentialStore, MemoryGatewayUserCredential def test_loader_keeps_curated_memory_in_explicit_curated_mode(tmp_path) -> None: @@ -14,7 +15,9 @@ def test_loader_keeps_curated_memory_in_explicit_curated_mode(tmp_path) -> None: loaded = EngineLoader(workspace=tmp_path, config=config).load() try: - assert loaded.memory_gateway_service is None + assert loaded.memory_gateway_config is None + assert loaded.memory_gateway_credentials is None + assert loaded.memory_gateway_service_factory is None assert loaded.curated_memory_store is not None assert loaded.memory_service is not None assert "memory" in loaded.tools @@ -26,22 +29,30 @@ def test_loader_keeps_curated_memory_in_explicit_curated_mode(tmp_path) -> None: def test_loader_adds_gateway_service_without_disabling_curated_memory(tmp_path) -> None: gateway_config = MemoryGatewayConfig( base_url="http://gateway.test", - user_id="gateway-user", - user_key="uk_secret", ) config = BeaverConfig( memory=MemoryConfig(mode="hybrid", explicit=True, gateway=gateway_config) ) + credential_store = MemoryGatewayCredentialStore(tmp_path / "memory_gateway_users.json") fake_gateway_service = object() loaded = EngineLoader( workspace=tmp_path, config=config, - memory_gateway_service=fake_gateway_service, + memory_gateway_credentials=credential_store, + memory_gateway_service_factory=lambda cfg, credential: fake_gateway_service, ).load() try: - assert loaded.memory_gateway_service is fake_gateway_service + assert loaded.memory_gateway_config == gateway_config + assert loaded.memory_gateway_credentials is credential_store + assert loaded.memory_gateway_service_factory is not None + assert ( + loaded.memory_gateway_service_factory( + MemoryGatewayUserCredential(user_id="gateway-user", user_key="uk_secret") + ) + is fake_gateway_service + ) assert loaded.curated_memory_store is not None assert loaded.memory_service is not None assert "memory" in loaded.tools @@ -60,7 +71,7 @@ def test_loader_implicit_hybrid_without_credentials_warns_and_degrades( loaded = EngineLoader(workspace=tmp_path, config=config).load() try: - assert loaded.memory_gateway_service is None + assert loaded.memory_gateway_config is None assert loaded.curated_memory_store is not None assert "memory" in loaded.tools assert "continuing with curated memory only" in caplog.text @@ -76,7 +87,7 @@ def test_loader_explicit_hybrid_without_credentials_fails_before_opening_session memory=MemoryConfig( mode="hybrid", explicit=True, - gateway=MemoryGatewayConfig(user_key="uk_super_secret"), + gateway=MemoryGatewayConfig(), ) ) @@ -89,4 +100,3 @@ def test_loader_explicit_hybrid_without_credentials_fails_before_opening_session EngineLoader(workspace=tmp_path, config=config).load() assert "Memory Gateway" in str(exc_info.value) - assert "uk_super_secret" not in str(exc_info.value) diff --git a/app-instance/backend/tests/unit/test_memory_gateway_registration.py b/app-instance/backend/tests/unit/test_memory_gateway_registration.py new file mode 100644 index 0000000..198f3c7 --- /dev/null +++ b/app-instance/backend/tests/unit/test_memory_gateway_registration.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import json +import logging + +from fastapi.testclient import TestClient + +from beaver.interfaces.web.app import create_app +from beaver.memory.gateway import ( + MemoryGatewayClientError, + MemoryGatewayCredentialStore, +) +from beaver.services.agent_service import AgentService + + +class FakeGatewayClient: + def __init__( + self, + *, + response: dict[str, str] | None = None, + error: MemoryGatewayClientError | None = None, + ) -> None: + self.response = response or {"user_id": "tom", "user_key": "uk_tom"} + self.error = error + self.calls: list[str] = [] + + async def create_user(self, user_id: str) -> dict[str, str]: + self.calls.append(user_id) + if self.error is not None: + raise self.error + return dict(self.response) + + +def _service(tmp_path) -> AgentService: + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({}), encoding="utf-8") + return AgentService(config_path=config_path) + + +def _write_memory_config(tmp_path) -> None: + memory_config_path = tmp_path / "memory-config.json" + memory_config_path.write_text( + json.dumps( + { + "memory": { + "mode": "hybrid", + "gateway": { + "baseUrl": "http://172.19.207.37:8010", + "appId": "default", + "projectId": "default", + "scope": ["current_chat", "resources", "all_user_memory"], + "topK": 8, + "timeoutSeconds": 10, + }, + } + } + ), + encoding="utf-8", + ) + + +def test_register_provisions_gateway_user_and_hides_key( + tmp_path, monkeypatch +) -> None: + auth_path = tmp_path / "web_auth_users.json" + users_path = tmp_path / "memory_gateway_users.json" + monkeypatch.setenv("BEAVER_AUTH_FILE", str(auth_path)) + monkeypatch.setenv("BEAVER_MEMORY_GATEWAY_USERS_PATH", str(users_path)) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(tmp_path / "memory-config.json")) + _write_memory_config(tmp_path) + + service = _service(tmp_path) + app = create_app(service=service, manage_service_lifecycle=False) + fake_client = FakeGatewayClient(response={"user_id": "tom", "user_key": "uk_tom"}) + app.state.memory_gateway_client_factory = lambda _config: fake_client + + with TestClient(app) as client: + response = client.post( + "/api/auth/register", + json={"username": "tom", "password": "pw"}, + ) + + assert response.status_code == 200 + assert fake_client.calls == ["tom"] + body = response.json() + assert "user_key" not in json.dumps(body) + assert MemoryGatewayCredentialStore(users_path).get("tom") is not None + assert MemoryGatewayCredentialStore(users_path).get("tom").user_key == "uk_tom" + service.close() + + +def test_register_keeps_local_user_and_logs_when_gateway_provisioning_fails( + tmp_path, monkeypatch, caplog +) -> None: + auth_path = tmp_path / "web_auth_users.json" + users_path = tmp_path / "memory_gateway_users.json" + monkeypatch.setenv("BEAVER_AUTH_FILE", str(auth_path)) + monkeypatch.setenv("BEAVER_MEMORY_GATEWAY_USERS_PATH", str(users_path)) + monkeypatch.setenv("BEAVER_MEMORY_CONFIG_PATH", str(tmp_path / "memory-config.json")) + _write_memory_config(tmp_path) + + service = _service(tmp_path) + app = create_app(service=service, manage_service_lifecycle=False) + app.state.memory_gateway_client_factory = lambda _config: FakeGatewayClient( + error=MemoryGatewayClientError("create_user", "network") + ) + + with caplog.at_level(logging.WARNING, logger="beaver.interfaces.web.app"): + with TestClient(app) as client: + response = client.post( + "/api/auth/register", + json={"username": "tom", "password": "pw"}, + ) + + assert response.status_code == 200 + auth_payload = json.loads(auth_path.read_text(encoding="utf-8")) + assert auth_payload == {"users": [{"username": "tom", "password": "pw"}]} + assert MemoryGatewayCredentialStore(users_path).get("tom") is None + assert "Memory Gateway user provisioning failed" in caplog.text + assert "operation=create_user" in caplog.text + assert "category=network" in caplog.text + assert "user_key" not in caplog.text + service.close() diff --git a/app-instance/backend/tests/unit/test_memory_gateway_service.py b/app-instance/backend/tests/unit/test_memory_gateway_service.py index 085dd2d..620fdb9 100644 --- a/app-instance/backend/tests/unit/test_memory_gateway_service.py +++ b/app-instance/backend/tests/unit/test_memory_gateway_service.py @@ -5,16 +5,18 @@ import json import httpx import pytest -from beaver.foundation.config import MemoryGatewayConfig -from beaver.integrations.memory_gateway import MemoryGatewayClient, MemoryGatewayClientError -from beaver.services.memory_gateway_service import MemoryGatewayService +from beaver.memory.gateway import ( + MemoryGatewayClient, + MemoryGatewayClientError, + MemoryGatewayConfig, + MemoryGatewayService, + MemoryGatewayUserCredential, +) def _config() -> MemoryGatewayConfig: return MemoryGatewayConfig( base_url="http://gateway.test", - user_id="gateway-user", - user_key="uk_super_secret", app_id="beaver", project_id="sandbox", scope=["current_chat", "resources"], @@ -23,6 +25,10 @@ def _config() -> MemoryGatewayConfig: ) +def _credential() -> MemoryGatewayUserCredential: + return MemoryGatewayUserCredential(user_id="gateway-user", user_key="uk_super_secret") + + @pytest.mark.asyncio async def test_client_uses_exact_gateway_paths_and_payloads() -> None: requests: list[httpx.Request] = [] @@ -113,7 +119,7 @@ async def test_recall_sanitizes_results_and_builds_reference_message() -> None: ] } ) - service = MemoryGatewayService(_config(), client=client) + service = MemoryGatewayService(_config(), _credential(), client=client) outcome = await service.recall_before_run(session_id="web:alpha", query="contract") @@ -146,6 +152,7 @@ async def test_recall_sanitizes_results_and_builds_reference_message() -> None: async def test_recall_rejects_malformed_results_shape() -> None: service = MemoryGatewayService( _config(), + _credential(), client=FakeGatewayClient(search_response={"results": {"not": "a list"}}), ) @@ -160,7 +167,7 @@ async def test_recall_rejects_malformed_results_shape() -> None: @pytest.mark.asyncio async def test_persist_after_run_adds_two_messages_then_flushes() -> None: client = FakeGatewayClient() - service = MemoryGatewayService(_config(), client=client) + service = MemoryGatewayService(_config(), _credential(), client=client) outcome = await service.persist_after_run( session_id="web:alpha", @@ -206,7 +213,7 @@ async def test_persist_after_run_adds_two_messages_then_flushes() -> None: async def test_add_failure_skips_flush() -> None: add_error = MemoryGatewayClientError("add", "http_status", status_code=503) client = FakeGatewayClient(add_error=add_error) - service = MemoryGatewayService(_config(), client=client) + service = MemoryGatewayService(_config(), _credential(), client=client) outcome = await service.persist_after_run( session_id="web:alpha", @@ -226,7 +233,7 @@ async def test_add_failure_skips_flush() -> None: async def test_flush_failure_preserves_successful_add() -> None: flush_error = MemoryGatewayClientError("flush", "network") client = FakeGatewayClient(flush_error=flush_error) - service = MemoryGatewayService(_config(), client=client) + service = MemoryGatewayService(_config(), _credential(), client=client) outcome = await service.persist_after_run( session_id="web:alpha", diff --git a/app-instance/backend/tests/unit/test_websocket_chat.py b/app-instance/backend/tests/unit/test_websocket_chat.py index dcf8bf1..d36fa96 100644 --- a/app-instance/backend/tests/unit/test_websocket_chat.py +++ b/app-instance/backend/tests/unit/test_websocket_chat.py @@ -88,6 +88,7 @@ def test_websocket_message_returns_chat_metadata_and_session_updated() -> None: "session_id": "web:alpha", "source": "websocket", "user_id": None, + "gateway_user_id": None, "title": None, "execution_context": None, "prompt_locale": "zh-Hant", @@ -134,6 +135,7 @@ def test_websocket_message_uses_direct_processing_when_loop_is_not_running() -> "session_id": "web:alpha", "source": "websocket", "user_id": None, + "gateway_user_id": None, "title": None, "execution_context": None, "prompt_locale": None, @@ -164,6 +166,7 @@ def test_rest_chat_uses_direct_processing_when_loop_is_not_running() -> None: "session_id": "web:alpha", "source": "web", "user_id": None, + "gateway_user_id": None, "title": None, "execution_context": None, "prompt_locale": "en", @@ -181,6 +184,72 @@ def test_rest_chat_uses_direct_processing_when_loop_is_not_running() -> None: assert response.json()["output_text"] == "echo:hello" +def test_rest_chat_uses_authenticated_user_for_gateway_identity() -> None: + service = DirectModeOnlyAgentService() + app = create_app(service=service, manage_service_lifecycle=False) + app.state.auth_tokens["token-1"] = "tom" + + with TestClient(app) as client: + response = client.post( + "/api/chat", + headers={"Authorization": "Bearer token-1"}, + json={"session_id": "web:alpha", "message": "hello", "user_id": "other"}, + ) + + assert response.status_code == 200 + assert service.calls == [ + { + "message": "hello", + "session_id": "web:alpha", + "source": "web", + "user_id": "other", + "gateway_user_id": "tom", + "title": None, + "execution_context": None, + "prompt_locale": None, + "model": None, + "provider_name": None, + "embedding_model": None, + "temperature": None, + "max_tokens": None, + "max_tool_iterations": None, + "fallback_target": None, + "auxiliary_target": None, + "embedding_target": None, + } + ] + + +def test_websocket_uses_authenticated_user_for_gateway_identity() -> None: + service = StubAgentService() + app = create_app(service=service, manage_service_lifecycle=False) + app.state.auth_tokens["token-1"] = "tom" + + with TestClient(app) as client: + with client.websocket_connect("/ws/web:alpha?token=token-1") as websocket: + websocket.send_json({"type": "message", "content": "hello", "user_id": "other"}) + assert websocket.receive_json() == {"type": "status", "status": "thinking"} + websocket.receive_json() + websocket.receive_json() + + assert service.calls == [ + { + "message": "hello", + "session_id": "web:alpha", + "source": "websocket", + "user_id": "other", + "gateway_user_id": "tom", + "title": None, + "execution_context": None, + "prompt_locale": None, + "model": None, + "provider_name": None, + "embedding_model": None, + "max_tool_iterations": None, + } + ] + + def test_websocket_empty_content_returns_error_without_runtime_call() -> None: service = StubAgentService() app = create_app(service=service, manage_service_lifecycle=False) diff --git a/app-instance/create-instance.sh b/app-instance/create-instance.sh index 4a98987..b4a969d 100755 --- a/app-instance/create-instance.sh +++ b/app-instance/create-instance.sh @@ -737,6 +737,7 @@ INSTANCE_ROOT="${INSTANCES_ROOT}/${INSTANCE_SLUG}" BEAVER_HOME="${INSTANCE_ROOT}/beaver-home" CONFIG_PATH="${BEAVER_HOME}/config.json" AUTH_USERS_PATH="${BEAVER_HOME}/web_auth_users.json" +MEMORY_GATEWAY_USERS_PATH="${BEAVER_HOME}/memory_gateway_users.json" RUNTIME_ENV_PATH="${BEAVER_HOME}/runtime.env" WORKSPACE_PATH="${BEAVER_HOME}/workspace" @@ -745,6 +746,8 @@ mkdir -p "$BEAVER_HOME" "$WORKSPACE_PATH" render_config_json "$CONFIG_PATH" render_auth_users_json "$AUTH_USERS_PATH" render_runtime_env_file "$RUNTIME_ENV_PATH" +printf '{\n "users": {}\n}\n' >"$MEMORY_GATEWAY_USERS_PATH" +chmod 600 "$MEMORY_GATEWAY_USERS_PATH" seed_initial_skills "$WORKSPACE_PATH" "$INITIAL_SKILLS_DIR" if [[ "$FORCE_BUILD" -eq 1 ]] || ! image_exists; then @@ -775,6 +778,7 @@ RUN_ARGS=( -e "BEAVER_CONFIG_PATH=/root/.beaver/config.json" -e "BEAVER_WORKSPACE=/root/.beaver/workspace" -e "BEAVER_AUTH_FILE=/root/.beaver/web_auth_users.json" + -e "BEAVER_MEMORY_GATEWAY_USERS_PATH=/root/.beaver/memory_gateway_users.json" -e "BEAVER_FRONTEND_PUBLIC_BASE_URL=${PUBLIC_URL}" -e "APP_PUBLIC_PORT=8080" -e "APP_FRONTEND_PORT=3000" diff --git a/app-instance/entrypoint.sh b/app-instance/entrypoint.sh index 17a31c6..6b1d9d4 100755 --- a/app-instance/entrypoint.sh +++ b/app-instance/entrypoint.sh @@ -11,6 +11,7 @@ BEAVER_HOME="${BEAVER_HOME:-/root/.beaver}" BEAVER_CONFIG_PATH="${BEAVER_CONFIG_PATH:-$BEAVER_HOME/config.json}" BEAVER_WORKSPACE="${BEAVER_WORKSPACE:-$BEAVER_HOME/workspace}" BEAVER_AUTH_FILE="${BEAVER_AUTH_FILE:-$BEAVER_HOME/web_auth_users.json}" +BEAVER_MEMORY_GATEWAY_USERS_PATH="${BEAVER_MEMORY_GATEWAY_USERS_PATH:-$BEAVER_HOME/memory_gateway_users.json}" BEAVER_RUNTIME_ENV_FILE="${BEAVER_RUNTIME_ENV_FILE:-$BEAVER_HOME/runtime.env}" BEAVER_INITIAL_SKILLS_DIR="${BEAVER_INITIAL_SKILLS_DIR:-/opt/app/initial-skills}" BEAVER_INITIAL_SKILLS_EXCLUDE="${BEAVER_INITIAL_SKILLS_EXCLUDE:-officebench-mcp}" @@ -111,6 +112,11 @@ trap cleanup EXIT INT TERM mkdir -p "$BEAVER_HOME" "$BEAVER_WORKSPACE" +if [[ ! -f "$BEAVER_MEMORY_GATEWAY_USERS_PATH" ]]; then + printf '{\n "users": {}\n}\n' >"$BEAVER_MEMORY_GATEWAY_USERS_PATH" + chmod 600 "$BEAVER_MEMORY_GATEWAY_USERS_PATH" +fi + if [[ -f "$BEAVER_RUNTIME_ENV_FILE" ]]; then set -a . "$BEAVER_RUNTIME_ENV_FILE" @@ -121,6 +127,7 @@ require_file "$BEAVER_CONFIG_PATH" "Missing Beaver config" seed_initial_skills "$BEAVER_INITIAL_SKILLS_DIR" "$BEAVER_WORKSPACE/skills" export BEAVER_AUTH_FILE +export BEAVER_MEMORY_GATEWAY_USERS_PATH export BEAVER_RUNTIME_ENV_FILE export BEAVER_HOME export BEAVER_CONFIG_PATH