diff --git a/.env.example b/.env.example index d8bbfea..5ab15b2 100644 --- a/.env.example +++ b/.env.example @@ -7,8 +7,8 @@ BEAVER_PROXY_CONTAINER_NAME=beaver-router-proxy BEAVER_DEPLOY_TOKEN=change-me BEAVER_AUTHZ_INTERNAL_TOKEN=change-me -BEAVER_SERVER_IP=203.0.113.10 -BEAVER_BASE_DOMAIN=203.0.113.10.nip.io +BEAVER_SERVER_IP=127.0.0.1 +BEAVER_BASE_DOMAIN=localhost BEAVER_PROVIDER=openai BEAVER_MODEL=openai/gpt-5 diff --git a/README.md b/README.md index db42c90..64aefae 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ export BEAVER_PROXY_CONTAINER_NAME=beaver-router-proxy export BEAVER_DEPLOY_TOKEN="$(openssl rand -hex 32)" export BEAVER_AUTHZ_INTERNAL_TOKEN="$(openssl rand -hex 32)" -export BEAVER_BASE_DOMAIN=127.0.0.1.nip.io +export BEAVER_BASE_DOMAIN=localhost export BEAVER_AUTHZ_URL='http://beaver-authz-service:19090' export BEAVER_DEPLOY_URL='http://beaver-deploy-control:8090' @@ -110,14 +110,14 @@ http://beaver-authz-service:19090 ```bash DEPLOY_PUBLIC_SCHEME=http -DEPLOY_PUBLIC_BASE_DOMAIN=127.0.0.1.nip.io +DEPLOY_PUBLIC_BASE_DOMAIN=localhost DEPLOY_PUBLIC_PORT=8088 ``` 本机测试时实例 URL 形如： ```text -http://alice.127.0.0.1.nip.io:8088 +http://alice.localhost:8088 ``` 正式 HTTPS 域名通常改成： diff --git a/app-instance/backend/beaver/engine/loader.py b/app-instance/backend/beaver/engine/loader.py index 86362f6..7be13c1 100644 --- a/app-instance/backend/beaver/engine/loader.py +++ b/app-instance/backend/beaver/engine/loader.py @@ -47,6 +47,12 @@ from beaver.tools.builtins import ( SkillsListTool, TerminalTool, TodoTool, + UserFilesCopyToWorkspaceTool, + UserFilesListTool, + UserFilesMkdirTool, + UserFilesPublishOutputTool, + UserFilesReadTool, + UserFilesWriteTool, WebFetchTool, WebSearchTool, WriteFileTool, @@ -222,6 +228,12 @@ class EngineLoader: ObjectBackedTool(SearchFilesTool()), ObjectBackedTool(WriteFileTool()), ObjectBackedTool(PatchFileTool()), + ObjectBackedTool(UserFilesListTool()), + ObjectBackedTool(UserFilesReadTool()), + ObjectBackedTool(UserFilesWriteTool()), + ObjectBackedTool(UserFilesMkdirTool()), + ObjectBackedTool(UserFilesCopyToWorkspaceTool()), + ObjectBackedTool(UserFilesPublishOutputTool()), ObjectBackedTool(WebFetchTool()), ObjectBackedTool(WebSearchTool()), ObjectBackedTool(TerminalTool()), diff --git a/app-instance/backend/beaver/engine/loop.py b/app-instance/backend/beaver/engine/loop.py index ba6ec5e..34683dc 100644 --- a/app-instance/backend/beaver/engine/loop.py +++ b/app-instance/backend/beaver/engine/loop.py @@ -621,11 +621,17 @@ class AgentLoop: "tool_registry": tool_registry, "skills_loader": skills_loader, "draft_service": getattr(loaded, "draft_service", None), + "beaver_config": loaded.config, + "task_id": task_id, + "run_id": resolved_run_id, **self.runtime_services, }, metadata={ "source": source, "agent_name": self.profile.name, + "session_id": resolved_session_id, + "task_id": task_id, + "run_id": resolved_run_id, }, ) diff --git a/app-instance/backend/beaver/engine/session/store.py b/app-instance/backend/beaver/engine/session/store.py index 68865fb..71ea10b 100644 --- a/app-instance/backend/beaver/engine/session/store.py +++ b/app-instance/backend/beaver/engine/session/store.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import os import sqlite3 import threading import time @@ -110,6 +111,12 @@ END; """ +def _sqlite_journal_mode() -> str: + requested = os.getenv("BEAVER_SQLITE_JOURNAL_MODE", "DELETE").strip().upper() + allowed = {"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "OFF", "WAL"} + return requested if requested in allowed else "DELETE" + + class SessionStore: """SQLite-backed session store.""" @@ -119,7 +126,9 @@ class SessionStore: self._lock = threading.Lock() self._conn = sqlite3.connect(str(self.db_path), check_same_thread=False, isolation_level=None) self._conn.row_factory = sqlite3.Row - self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA mmap_size=0") + self._conn.execute("PRAGMA busy_timeout=5000") + self._conn.execute(f"PRAGMA journal_mode={_sqlite_journal_mode()}") self._conn.execute("PRAGMA foreign_keys=ON") self._init_schema() diff --git a/app-instance/backend/beaver/foundation/config/loader.py b/app-instance/backend/beaver/foundation/config/loader.py index ce75a8b..e55fd50 100644 --- a/app-instance/backend/beaver/foundation/config/loader.py +++ b/app-instance/backend/beaver/foundation/config/loader.py @@ -20,7 +20,7 @@ from .schema import ( ) LOCAL_MCP_CATEGORIES: dict[str, dict[str, str]] = { - "local_filesystem_mcp": {"category": "filesystem", "display_name": "本地文件工具"}, + "local_filesystem_mcp": {"category": "filesystem", "display_name": "个人智能体文件系统工具"}, "local_runtime_mcp": {"category": "runtime", "display_name": "本地运行工具"}, "local_memory_mcp": {"category": "memory", "display_name": "本地记忆工具"}, "local_skills_mcp": {"category": "skills", "display_name": "本地技能工具"}, diff --git a/app-instance/backend/beaver/integrations/authz/client.py b/app-instance/backend/beaver/integrations/authz/client.py index 411a718..f11cec0 100644 --- a/app-instance/backend/beaver/integrations/authz/client.py +++ b/app-instance/backend/beaver/integrations/authz/client.py @@ -109,3 +109,15 @@ class AuthzClient: async def delete_outlook_settings(self, backend_id: str) -> dict[str, Any]: data = await self._request("DELETE", f"/backends/{backend_id}/settings/outlook") return data if isinstance(data, dict) else {} + + async def get_minio_settings(self, backend_id: str) -> dict[str, Any]: + data = await self._request("GET", f"/backends/{backend_id}/settings/minio") + return data if isinstance(data, dict) else {} + + async def set_minio_settings(self, backend_id: str, payload: dict[str, Any]) -> dict[str, Any]: + data = await self._request("POST", f"/backends/{backend_id}/settings/minio", json_body=payload) + return data if isinstance(data, dict) else {} + + async def delete_minio_settings(self, backend_id: str) -> dict[str, Any]: + data = await self._request("DELETE", f"/backends/{backend_id}/settings/minio") + return data if isinstance(data, dict) else {} diff --git a/app-instance/backend/beaver/interfaces/mcp/tools_server.py b/app-instance/backend/beaver/interfaces/mcp/tools_server.py index a333b10..bb87fb2 100644 --- a/app-instance/backend/beaver/interfaces/mcp/tools_server.py +++ b/app-instance/backend/beaver/interfaces/mcp/tools_server.py @@ -27,12 +27,8 @@ from beaver.tools.builtins import ( CronTool, DelegateTool, ExecuteCodeTool, - ListDirectoryTool, MemoryTool, - PatchFileTool, ProcessTool, - ReadFileTool, - SearchFilesTool, SendMessageTool, SkillManageTool, SkillViewTool, @@ -40,6 +36,12 @@ from beaver.tools.builtins import ( SpawnTool, TerminalTool, TodoTool, + UserFilesCopyToWorkspaceTool, + UserFilesListTool, + UserFilesMkdirTool, + UserFilesPublishOutputTool, + UserFilesReadTool, + UserFilesWriteTool, WebFetchTool, WebSearchTool, WriteFileTool, @@ -47,7 +49,7 @@ from beaver.tools.builtins import ( LOCAL_TOOL_CATEGORIES = { - "filesystem": "Beaver Local Filesystem Tools", + "filesystem": "Beaver Personal Agent Filesystem Tools", "runtime": "Beaver Local Runtime Tools", "memory": "Beaver Local Memory Tools", "skills": "Beaver Local Skills Tools", @@ -84,11 +86,12 @@ def _category_tools(category: str, workspace: Path) -> tuple[list[BaseTool], Too if category == "filesystem": tools: list[BaseTool] = [ - ObjectBackedTool(ListDirectoryTool()), - ObjectBackedTool(ReadFileTool()), - ObjectBackedTool(SearchFilesTool()), - ObjectBackedTool(WriteFileTool()), - ObjectBackedTool(PatchFileTool()), + ObjectBackedTool(UserFilesListTool()), + ObjectBackedTool(UserFilesReadTool()), + ObjectBackedTool(UserFilesWriteTool()), + ObjectBackedTool(UserFilesMkdirTool()), + ObjectBackedTool(UserFilesCopyToWorkspaceTool()), + ObjectBackedTool(UserFilesPublishOutputTool()), ] elif category == "runtime": tools = [ diff --git a/app-instance/backend/beaver/interfaces/web/app.py b/app-instance/backend/beaver/interfaces/web/app.py index cc06fbc..e2d1704 100644 --- a/app-instance/backend/beaver/interfaces/web/app.py +++ b/app-instance/backend/beaver/interfaces/web/app.py @@ -24,6 +24,19 @@ from beaver.integrations.mcp import MCPConnectionManager from beaver.services.agent_service import NOTIFICATION_SESSION_ID, AgentService from beaver.services.cron_service import CronService, schedule_from_api from beaver.services.skillhub_service import SkillHubService +from beaver.services.user_files import ( + USER_FILE_ROOTS, + UserFileError, + UserFileNotFoundError, + UserFilePathError, + UserFileSizeError, + UserFileService, +) +from beaver.services.user_file_resolver import ( + UserFileConfigurationError, + UserFileStorageResolver, + build_file_auth_context, +) from beaver.skills.learning import SkillLearningWorker, SkillLearningWorkerConfig from beaver.skills.catalog.utils import parse_frontmatter @@ -306,6 +319,28 @@ def create_app( app.state.handoff_codes = {} app.state.auth_file = Path(os.getenv("BEAVER_AUTH_FILE") or "") max_file_size = 50 * 1024 * 1024 + max_user_file_upload_size = _int_env("BEAVER_USER_FILES_MAX_UPLOAD_BYTES", 5 * 1024 * 1024 * 1024) + user_file_upload_part_size = _int_env("BEAVER_USER_FILES_UPLOAD_PART_SIZE", 10 * 1024 * 1024) + + def _user_file_resolver(request: Request, authorization: str | None) -> UserFileStorageResolver: + username = _require_web_user(app, authorization) + loaded = get_agent_service(request).create_loop().boot() + auth_context = build_file_auth_context(username=username, config=loaded.config) + return UserFileStorageResolver(config=loaded.config, workspace=loaded.workspace, auth_context=auth_context) + + async def _user_file_service(request: Request, authorization: str | None) -> UserFileService: + return await _user_file_resolver(request, authorization).service() + + def _user_file_http_error(exc: UserFileError) -> HTTPException: + if isinstance(exc, UserFileNotFoundError): + return HTTPException(status_code=404, detail=str(exc) or "File not found") + if isinstance(exc, UserFilePathError): + return HTTPException(status_code=400, detail=str(exc) or "Invalid path") + if isinstance(exc, UserFileSizeError): + return HTTPException(status_code=413, detail=str(exc) or "File too large") + if isinstance(exc, UserFileConfigurationError): + return HTTPException(status_code=503, detail=str(exc) or "User file storage is not configured") + return HTTPException(status_code=400, detail=str(exc) or "User file operation failed") @app.get("/api/ping", response_model=WebStatusResponse) async def ping(request: Request) -> WebStatusResponse: @@ -747,6 +782,101 @@ def create_app( return {"ok": True} raise HTTPException(status_code=404, detail="File not found") + @app.get("/api/user-files/status") + async def user_files_status( + request: Request, + authorization: str | None = Header(default=None), + ) -> dict[str, Any]: + return (await _user_file_resolver(request, authorization).status()).to_dict() + + @app.get("/api/user-files/browse") + async def browse_user_files( + request: Request, + path: str = "", + authorization: str | None = Header(default=None), + ) -> dict[str, Any]: + try: + return await (await _user_file_service(request, authorization)).browse(path) + except UserFileError as exc: + raise _user_file_http_error(exc) from exc + + @app.get("/api/user-files/download") + async def download_user_file( + path: str, + request: Request, + authorization: str | None = Header(default=None), + ) -> Response: + try: + content = await (await _user_file_service(request, authorization)).download(path) + except UserFileError as exc: + raise _user_file_http_error(exc) from exc + disposition = "inline" if content.content_type.startswith("image/") else "attachment" + return Response( + content=content.content, + media_type=content.content_type, + headers={"Content-Disposition": content_disposition(disposition, content.name)}, + ) + + @app.get("/api/user-files/preview") + async def preview_user_file( + path: str, + request: Request, + authorization: str | None = Header(default=None), + ) -> dict[str, Any]: + try: + return await (await _user_file_service(request, authorization)).preview(path) + except UserFileError as exc: + raise _user_file_http_error(exc) from exc + + @app.post("/api/user-files/upload") + async def upload_user_file( + request: Request, + file: UploadFile = File(...), + path: str = Form("uploads"), + authorization: str | None = Header(default=None), + ) -> dict[str, Any]: + if not file.filename: + raise HTTPException(status_code=400, detail="No filename provided") + file_size = getattr(file, "size", None) + if isinstance(file_size, int) and file_size > max_user_file_upload_size: + raise HTTPException(status_code=413, detail=f"File too large (max {_human_upload_size(max_user_file_upload_size)})") + try: + return await (await _user_file_service(request, authorization)).upload_stream( + path, + file.filename, + file.file, + content_type=file.content_type or "application/octet-stream", + max_bytes=max_user_file_upload_size, + part_size=user_file_upload_part_size, + ) + except UserFileError as exc: + raise _user_file_http_error(exc) from exc + + @app.delete("/api/user-files/delete") + async def delete_user_file( + path: str, + request: Request, + authorization: str | None = Header(default=None), + ) -> dict[str, bool]: + try: + removed = await (await _user_file_service(request, authorization)).delete(path) + except UserFileError as exc: + raise _user_file_http_error(exc) from exc + if removed: + return {"ok": True} + raise HTTPException(status_code=404, detail="Path not found") + + @app.post("/api/user-files/mkdir") + async def create_user_file_directory( + path: str, + request: Request, + authorization: str | None = Header(default=None), + ) -> dict[str, Any]: + try: + return await (await _user_file_service(request, authorization)).mkdir(path) + except UserFileError as exc: + raise _user_file_http_error(exc) from exc + @app.get("/api/workspace/browse") async def browse_workspace_dir(request: Request, path: str = "") -> dict[str, Any]: loaded = get_agent_service(request).create_loop().boot() @@ -2576,6 +2706,27 @@ def _handoff_replay_window_seconds() -> int: return 15 +def _int_env(name: str, default: int) -> int: + raw = os.getenv(name, "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError: + return default + return value if value > 0 else default + + +def _human_upload_size(size: int) -> str: + units = ("B", "KB", "MB", "GB", "TB") + value = float(size) + for unit in units: + if value < 1024 or unit == units[-1]: + return f"{value:.0f}{unit}" if unit == "B" else f"{value:.1f}{unit}" + value /= 1024 + return f"{size}B" + + def _prune_handoff_codes(app: FastAPI) -> None: now = time.time() replay_window = _handoff_replay_window_seconds() diff --git a/app-instance/backend/beaver/services/user_file_resolver.py b/app-instance/backend/beaver/services/user_file_resolver.py new file mode 100644 index 0000000..9563980 --- /dev/null +++ b/app-instance/backend/beaver/services/user_file_resolver.py @@ -0,0 +1,201 @@ +"""Resolve the user-visible file system for web and agent callers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import os +from pathlib import Path +from typing import Any + +import httpx + +from beaver.foundation.config.schema import BeaverConfig + +from .user_files import ( + LocalUserFileStorage, + MinIOStorageConfig, + MinIOUserFileStorage, + USER_FILE_ROOTS, + UserFileError, + UserFileService, +) + + +class UserFileConfigurationError(UserFileError): + """Raised when user file storage is not configured for this backend.""" + + +@dataclass(slots=True) +class FileAuthContext: + """Authenticated identity used by the personal file system boundary.""" + + username: str + backend_id: str + storage_namespace: str + user_id: str | None = None + scopes: tuple[str, ...] = field(default_factory=tuple) + auth_source: str = "beaver-web-token" + + +@dataclass(slots=True) +class UserFileStorageStatus: + configured: bool + storage_mode: str + roots: list[str] + workspace_visible: bool = False + detail: str | None = None + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "configured": self.configured, + "storage_mode": self.storage_mode, + "roots": self.roots, + "workspace_visible": self.workspace_visible, + } + if self.detail: + payload["detail"] = self.detail + return payload + + +class UserFileStorageResolver: + """Build `UserFileService` from the current Beaver identity and config.""" + + def __init__( + self, + *, + config: BeaverConfig, + workspace: Path, + auth_context: FileAuthContext, + ) -> None: + self.config = config + self.workspace = Path(workspace) + self.auth_context = auth_context + + async def service(self) -> UserFileService: + mode = _storage_mode(self.config) + if mode == "local": + return UserFileService(LocalUserFileStorage(self.workspace / "user_files")) + settings = await self._load_minio_settings() + return UserFileService( + MinIOUserFileStorage( + MinIOStorageConfig( + endpoint=str(settings.get("endpoint") or ""), + access_key=str(settings.get("access_key") or ""), + secret_key=str(settings.get("secret_key") or ""), + bucket=str(settings.get("bucket") or ""), + secure=bool(settings.get("secure", False)), + region=_clean_optional(settings.get("region")), + namespace=str(settings.get("namespace") or self.auth_context.storage_namespace), + ) + ) + ) + + async def status(self) -> UserFileStorageStatus: + mode = _storage_mode(self.config) + if mode == "local": + return UserFileStorageStatus( + configured=True, + storage_mode="local", + roots=list(USER_FILE_ROOTS), + workspace_visible=False, + ) + try: + await self._load_minio_settings() + except UserFileConfigurationError as exc: + return UserFileStorageStatus( + configured=False, + storage_mode="object", + roots=list(USER_FILE_ROOTS), + workspace_visible=False, + detail=str(exc), + ) + return UserFileStorageStatus( + configured=True, + storage_mode="object", + roots=list(USER_FILE_ROOTS), + workspace_visible=False, + ) + + async def _load_minio_settings(self) -> dict[str, Any]: + backend_id = self.auth_context.backend_id.strip() + if not backend_id: + raise UserFileConfigurationError("User file storage backend identity is not configured") + base_url = self.config.authz.base_url.strip() + if not (self.config.authz.enabled and base_url): + raise UserFileConfigurationError("AuthZ is required for deployed user file storage") + token = ( + os.getenv("BEAVER_AUTHZ_INTERNAL_TOKEN", "").strip() + or os.getenv("AUTHZ_INTERNAL_TOKEN", "").strip() + ) + if not token: + raise UserFileConfigurationError("AuthZ internal token is not configured for user file storage") + try: + async with httpx.AsyncClient( + timeout=self.config.authz.request_timeout_seconds, + follow_redirects=True, + trust_env=False, + ) as client: + response = await client.get( + f"{base_url.rstrip('/')}/internal/backends/{backend_id}/settings/minio", + headers={"Authorization": f"Bearer {token}"}, + ) + except httpx.HTTPError as exc: + raise UserFileConfigurationError(f"Unable to load user file storage settings: {exc}") from exc + if response.status_code == 404: + raise UserFileConfigurationError("MinIO user file storage is not configured") + if response.is_error: + raise UserFileConfigurationError( + f"Unable to load user file storage settings: HTTP {response.status_code}" + ) + payload = response.json() + if not isinstance(payload, dict): + raise UserFileConfigurationError("Invalid MinIO settings response") + if not all(str(payload.get(key) or "").strip() for key in ("endpoint", "access_key", "secret_key", "bucket")): + raise UserFileConfigurationError("MinIO user file storage settings are incomplete") + payload.setdefault("namespace", self.auth_context.storage_namespace) + return payload + + +def build_file_auth_context( + *, + username: str, + config: BeaverConfig, + user_id: str | None = None, + scopes: tuple[str, ...] = (), + auth_source: str = "beaver-web-token", +) -> FileAuthContext: + backend_id = ( + config.backend_identity.backend_id.strip() + or os.getenv("BEAVER_BACKEND_IDENTITY__BACKEND_ID", "").strip() + or username.strip() + ) + namespace = default_user_file_namespace(backend_id) + return FileAuthContext( + username=username.strip(), + backend_id=backend_id, + storage_namespace=namespace, + user_id=user_id, + scopes=scopes, + auth_source=auth_source, + ) + + +def default_user_file_namespace(backend_id: str) -> str: + cleaned = backend_id.strip().strip("/") + return f"users/{cleaned}" if cleaned else "users/unconfigured" + + +def _storage_mode(config: BeaverConfig) -> str: + raw = os.getenv("BEAVER_USER_FILES_STORAGE_MODE", "").strip().lower() + if raw in {"local", "dev-local", "development"}: + return "local" + if raw in {"minio", "object", "object-storage"}: + return "minio" + if config.authz.enabled and config.authz.base_url.strip() and config.backend_identity.backend_id.strip(): + return "minio" + return "local" + + +def _clean_optional(value: Any) -> str | None: + text = str(value or "").strip() + return text or None diff --git a/app-instance/backend/beaver/services/user_files.py b/app-instance/backend/beaver/services/user_files.py new file mode 100644 index 0000000..9052fcc --- /dev/null +++ b/app-instance/backend/beaver/services/user_files.py @@ -0,0 +1,630 @@ +"""User-visible file system service. + +This module owns the personal file-system boundary exposed to users and +agents. Storage backends can change, but callers see only virtual paths under +fixed roots. +""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timezone +from io import BytesIO +import mimetypes +from pathlib import Path, PurePosixPath +import shutil +import tempfile +from typing import Protocol + + +USER_FILE_ROOTS = ("uploads", "outputs", "shared", "tasks") +MAX_PREVIEW_BYTES = 1024 * 1024 +AGENT_UPLOADS_ERROR = "uploads/ is user-provided input storage; agents may read it but must not write it" +AGENT_DELETE_ERROR = "agents cannot delete user-visible files; use the Files page or user-side APIs" + + +class UserFileError(ValueError): + """Base error for user file operations.""" + + +class UserFilePathError(UserFileError): + """Raised when a user file path violates the virtual path policy.""" + + +class UserFileNotFoundError(UserFileError): + """Raised when a user file path does not exist.""" + + +class UserFileSizeError(UserFileError): + """Raised when a user file upload exceeds configured limits.""" + + +@dataclass(frozen=True, slots=True) +class AgentUserFilePolicy: + task_id: str | None = None + fallback_scope: str = "interactive" + + @property + def task_namespace(self) -> str: + if self.task_id: + return f"tasks/{self.task_id}" + scope = _safe_scope(self.fallback_scope) + return f"tasks/interactive/{scope}" + + def validate_read(self, path: str) -> str: + return normalize_user_path(path, allow_root=False) + + def validate_write(self, path: str) -> str: + normalized = normalize_user_path(path, allow_root=False) + root = normalized.split("/", 1)[0] + if root == "uploads": + raise UserFilePathError(AGENT_UPLOADS_ERROR) + if root == "tasks": + self._validate_task_namespace(normalized) + return normalized + + def validate_mkdir(self, path: str) -> str: + return self.validate_write(path) + + def validate_delete(self, path: str) -> str: + normalize_user_path(path, allow_root=False) + raise UserFilePathError(AGENT_DELETE_ERROR) + + def _validate_task_namespace(self, normalized: str) -> None: + namespace = self.task_namespace + if normalized == "tasks" or not normalized.startswith(f"{namespace}/"): + raise UserFilePathError(f"Agent task files must be written under {namespace}/") + + +@dataclass(slots=True) +class UserFileEntry: + name: str + path: str + type: str + size: int | None = None + content_type: str | None = None + modified: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "name": self.name, + "path": self.path, + "type": self.type, + "size": self.size, + "content_type": self.content_type, + "modified": self.modified, + } + + +@dataclass(slots=True) +class UserFileContent: + name: str + path: str + size: int + content_type: str + modified: str | None + content: bytes + + +@dataclass(slots=True) +class UserFilePreview: + name: str + path: str + size: int + content_type: str + modified: str | None + is_binary: bool + is_truncated: bool + content: str | None + + def to_dict(self) -> dict[str, object]: + return { + "name": self.name, + "path": self.path, + "size": self.size, + "content_type": self.content_type, + "modified": self.modified, + "is_binary": self.is_binary, + "is_truncated": self.is_truncated, + "content": self.content, + } + + +class UserFileStorage(Protocol): + async def list_dir(self, path: str) -> list[UserFileEntry]: + ... + + async def read_file(self, path: str, *, max_bytes: int | None = None) -> UserFileContent: + ... + + async def write_file(self, path: str, content: bytes, *, content_type: str) -> UserFileEntry: + ... + + async def write_file_stream( + self, + path: str, + stream: object, + *, + content_type: str, + max_bytes: int | None = None, + part_size: int = 10 * 1024 * 1024, + ) -> UserFileEntry: + ... + + async def delete_path(self, path: str) -> bool: + ... + + async def mkdir(self, path: str) -> UserFileEntry: + ... + + +class UserFileService: + def __init__(self, storage: UserFileStorage) -> None: + self.storage = storage + + async def browse(self, path: str = "") -> dict[str, object]: + normalized = normalize_user_path(path, allow_root=True) + if normalized == "": + return { + "path": "", + "items": [ + UserFileEntry(name=root, path=root, type="directory").to_dict() + for root in USER_FILE_ROOTS + ], + } + entries = await self.storage.list_dir(normalized) + return {"path": normalized, "items": [entry.to_dict() for entry in entries]} + + async def upload(self, directory: str, filename: str, content: bytes, *, content_type: str) -> dict[str, object]: + if not is_safe_filename(filename): + raise UserFilePathError("Invalid filename") + target = normalize_user_path(_join_user_path(directory, filename), allow_root=False) + return (await self.storage.write_file(target, content, content_type=content_type)).to_dict() + + async def upload_stream( + self, + directory: str, + filename: str, + stream: object, + *, + content_type: str, + max_bytes: int | None = None, + part_size: int = 10 * 1024 * 1024, + ) -> dict[str, object]: + if not is_safe_filename(filename): + raise UserFilePathError("Invalid filename") + target = normalize_user_path(_join_user_path(directory, filename), allow_root=False) + return ( + await self.storage.write_file_stream( + target, + stream, + content_type=content_type, + max_bytes=max_bytes, + part_size=part_size, + ) + ).to_dict() + + async def write_file(self, path: str, content: bytes | str, *, content_type: str = "text/plain") -> dict[str, object]: + normalized = normalize_user_path(path, allow_root=False) + raw = content.encode("utf-8") if isinstance(content, str) else bytes(content) + return (await self.storage.write_file(normalized, raw, content_type=content_type)).to_dict() + + async def download(self, path: str) -> UserFileContent: + return await self.storage.read_file(normalize_user_path(path, allow_root=False)) + + async def preview(self, path: str, *, max_bytes: int = MAX_PREVIEW_BYTES) -> dict[str, object]: + content = await self.storage.read_file(normalize_user_path(path, allow_root=False), max_bytes=max_bytes) + is_binary = _is_probably_binary(content.content, content.content_type) + text = None if is_binary else content.content.decode("utf-8", errors="replace") + return UserFilePreview( + name=content.name, + path=content.path, + size=content.size, + content_type=content.content_type, + modified=content.modified, + is_binary=is_binary, + is_truncated=content.size > len(content.content), + content=text, + ).to_dict() + + async def delete(self, path: str) -> bool: + normalized = normalize_user_path(path, allow_root=False) + if normalized in USER_FILE_ROOTS: + raise UserFilePathError("Cannot delete virtual root folders") + return await self.storage.delete_path(normalized) + + async def mkdir(self, path: str) -> dict[str, object]: + normalized = normalize_user_path(path, allow_root=False) + if normalized in USER_FILE_ROOTS: + raise UserFilePathError("Virtual root folders already exist") + return (await self.storage.mkdir(normalized)).to_dict() + + +class LocalUserFileStorage: + """Filesystem-backed storage adapter for tests and local development.""" + + def __init__(self, root: Path) -> None: + self.root = Path(root).expanduser().resolve() + self.root.mkdir(parents=True, exist_ok=True) + for name in USER_FILE_ROOTS: + (self.root / name).mkdir(parents=True, exist_ok=True) + + async def list_dir(self, path: str) -> list[UserFileEntry]: + target = self._path(path) + if not target.exists(): + target.mkdir(parents=True, exist_ok=True) + if not target.is_dir(): + raise UserFilePathError("Path is not a directory") + entries: list[UserFileEntry] = [] + for child in sorted(target.iterdir(), key=lambda item: (not item.is_dir(), item.name.lower())): + if child.name.startswith("."): + continue + entries.append(self._entry(child)) + return entries + + async def read_file(self, path: str, *, max_bytes: int | None = None) -> UserFileContent: + target = self._path(path) + if not target.is_file(): + raise UserFileNotFoundError("File not found") + raw = target.read_bytes() + selected = raw[:max_bytes] if max_bytes is not None else raw + stat = target.stat() + content_type, _ = mimetypes.guess_type(target.name) + return UserFileContent( + name=target.name, + path=self._relative(target), + size=stat.st_size, + content_type=content_type or "application/octet-stream", + modified=_iso_from_timestamp(stat.st_mtime), + content=selected, + ) + + async def write_file(self, path: str, content: bytes, *, content_type: str) -> UserFileEntry: + target = self._path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + return self._entry(target, content_type=content_type) + + async def write_file_stream( + self, + path: str, + stream: object, + *, + content_type: str, + max_bytes: int | None = None, + part_size: int = 10 * 1024 * 1024, + ) -> UserFileEntry: + target = self._path(path) + target.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".tmp", dir=target.parent) + tmp_path = Path(tmp_name) + total = 0 + try: + with open(fd, "wb", closefd=True) as output: + while True: + chunk = stream.read(part_size) # type: ignore[attr-defined] + if not chunk: + break + total += len(chunk) + if max_bytes is not None and total > max_bytes: + raise UserFileSizeError(_size_error(max_bytes)) + output.write(chunk) + tmp_path.replace(target) + except Exception: + with suppress(FileNotFoundError): + tmp_path.unlink() + raise + return self._entry(target, content_type=content_type) + + async def delete_path(self, path: str) -> bool: + target = self._path(path) + if not target.exists(): + return False + if target.is_dir(): + shutil.rmtree(target) + else: + target.unlink() + return True + + async def mkdir(self, path: str) -> UserFileEntry: + target = self._path(path) + target.mkdir(parents=True, exist_ok=True) + return self._entry(target) + + def _path(self, path: str) -> Path: + normalized = normalize_user_path(path, allow_root=False) + target = (self.root / normalized).resolve() + try: + target.relative_to(self.root) + except ValueError as exc: + raise UserFilePathError("Path escapes user file root") from exc + return target + + def _relative(self, path: Path) -> str: + return path.relative_to(self.root).as_posix() + + def _entry(self, path: Path, *, content_type: str | None = None) -> UserFileEntry: + stat = path.stat() + guessed_type, _ = mimetypes.guess_type(path.name) + return UserFileEntry( + name=path.name, + path=self._relative(path), + type="directory" if path.is_dir() else "file", + size=None if path.is_dir() else stat.st_size, + content_type=None if path.is_dir() else (content_type or guessed_type or "application/octet-stream"), + modified=_iso_from_timestamp(stat.st_mtime), + ) + + +@dataclass(slots=True) +class MinIOStorageConfig: + endpoint: str + access_key: str + secret_key: str + bucket: str + secure: bool = False + region: str | None = None + namespace: str = "" + + +class MinIOUserFileStorage: + """MinIO-backed user file storage adapter.""" + + def __init__(self, config: MinIOStorageConfig) -> None: + if not config.endpoint or not config.access_key or not config.secret_key or not config.bucket: + raise ValueError("MinIO storage requires endpoint, access key, secret key, and bucket") + from minio import Minio + + self.config = config + self.client = Minio( + endpoint=config.endpoint, + access_key=config.access_key, + secret_key=config.secret_key, + secure=config.secure, + region=config.region, + ) + + async def list_dir(self, path: str) -> list[UserFileEntry]: + prefix = self._object_prefix(path) + objects = self.client.list_objects(self.config.bucket, prefix=prefix, recursive=False) + entries: list[UserFileEntry] = [] + for obj in objects: + object_name = str(obj.object_name or "") + user_path = self._user_path(object_name) + if not user_path or user_path == path or user_path.endswith("/.keep"): + continue + trimmed = user_path.rstrip("/") + name = PurePosixPath(trimmed).name + is_dir = bool(getattr(obj, "is_dir", False)) or object_name.endswith("/") + entries.append( + UserFileEntry( + name=name, + path=trimmed, + type="directory" if is_dir else "file", + size=None if is_dir else getattr(obj, "size", None), + content_type=None if is_dir else "application/octet-stream", + modified=obj.last_modified.isoformat() if getattr(obj, "last_modified", None) else None, + ) + ) + return sorted(entries, key=lambda item: (item.type != "directory", item.name.lower())) + + async def read_file(self, path: str, *, max_bytes: int | None = None) -> UserFileContent: + object_name = self._object_name(path) + try: + stat = self.client.stat_object(self.config.bucket, object_name) + if max_bytes is None: + response = self.client.get_object(self.config.bucket, object_name) + else: + response = self.client.get_object(self.config.bucket, object_name, length=max_bytes) + raw = response.read() + response.close() + response.release_conn() + except Exception as exc: + raise UserFileNotFoundError("File not found") from exc + return UserFileContent( + name=PurePosixPath(path).name, + path=path, + size=int(stat.size or len(raw)), + content_type=stat.content_type or "application/octet-stream", + modified=stat.last_modified.isoformat() if stat.last_modified else None, + content=raw, + ) + + async def write_file(self, path: str, content: bytes, *, content_type: str) -> UserFileEntry: + object_name = self._object_name(path) + result = self.client.put_object( + self.config.bucket, + object_name, + BytesIO(content), + length=len(content), + content_type=content_type, + ) + return UserFileEntry( + name=PurePosixPath(path).name, + path=path, + type="file", + size=len(content), + content_type=content_type, + modified=datetime.now(timezone.utc).isoformat(), + ) + + async def write_file_stream( + self, + path: str, + stream: object, + *, + content_type: str, + max_bytes: int | None = None, + part_size: int = 10 * 1024 * 1024, + ) -> UserFileEntry: + object_name = self._object_name(path) + reader = _LimitedReadStream(stream, max_bytes=max_bytes) + try: + self.client.put_object( + self.config.bucket, + object_name, + reader, + length=-1, + part_size=max(5 * 1024 * 1024, part_size), + content_type=content_type, + ) + except UserFileSizeError: + try: + self.client.remove_object(self.config.bucket, object_name) + except Exception: + pass + raise + return UserFileEntry( + name=PurePosixPath(path).name, + path=path, + type="file", + size=reader.bytes_read, + content_type=content_type, + modified=datetime.now(timezone.utc).isoformat(), + ) + + async def delete_path(self, path: str) -> bool: + object_name = self._object_name(path) + removed = False + try: + self.client.remove_object(self.config.bucket, object_name) + removed = True + except Exception: + pass + prefix = f"{object_name.rstrip('/')}/" + for obj in self.client.list_objects(self.config.bucket, prefix=prefix, recursive=True): + self.client.remove_object(self.config.bucket, str(obj.object_name)) + removed = True + return removed + + async def mkdir(self, path: str) -> UserFileEntry: + object_name = f"{self._object_name(path).rstrip('/')}/.keep" + self.client.put_object( + self.config.bucket, + object_name, + BytesIO(b""), + length=0, + content_type="application/x-directory", + ) + return UserFileEntry( + name=PurePosixPath(path).name, + path=path, + type="directory", + size=None, + modified=datetime.now(timezone.utc).isoformat(), + ) + + def _namespace(self) -> str: + return self.config.namespace.strip("/") + + def _object_name(self, path: str) -> str: + normalized = normalize_user_path(path, allow_root=False) + namespace = self._namespace() + object_name = f"{namespace}/{normalized}" if namespace else normalized + if object_name.startswith("/") or "/../" in f"/{object_name}/": + raise UserFilePathError("Object path escapes namespace") + return object_name + + def _object_prefix(self, path: str) -> str: + return f"{self._object_name(path).rstrip('/')}/" + + def _user_path(self, object_name: str) -> str: + namespace = self._namespace() + if namespace: + prefix = f"{namespace}/" + if not object_name.startswith(prefix): + raise UserFilePathError("Object path escapes namespace") + return object_name[len(prefix) :] + return object_name + + +def normalize_user_path(path: str | None, *, allow_root: bool) -> str: + original = (path or "").replace("\\", "/").strip() + if original.startswith("/"): + raise UserFilePathError("Absolute paths are not allowed") + raw = original.strip("/") + if raw == "": + if allow_root: + return "" + raise UserFilePathError("Path is required") + posix = PurePosixPath(raw) + if posix.is_absolute(): + raise UserFilePathError("Absolute paths are not allowed") + parts = [part for part in posix.parts if part not in ("", ".")] + if any(part == ".." for part in parts): + raise UserFilePathError("Parent-directory traversal is not allowed") + if any(part.startswith(".") for part in parts): + raise UserFilePathError("Hidden implementation paths are not allowed") + if not parts or parts[0] not in USER_FILE_ROOTS: + raise UserFilePathError("Path must be under uploads, outputs, shared, or tasks") + return "/".join(parts) + + +def is_safe_filename(filename: str) -> bool: + return bool(filename) and "/" not in filename and "\\" not in filename and not filename.startswith(".") + + +def _join_user_path(directory: str, filename: str) -> str: + normalized_dir = normalize_user_path(directory, allow_root=False) + return f"{normalized_dir.rstrip('/')}/{filename}" + + +def _is_probably_binary(raw: bytes, content_type: str) -> bool: + if content_type.startswith("text/") or content_type in { + "application/json", + "application/javascript", + "application/xml", + "application/x-yaml", + }: + return False + if not raw: + return False + if b"\x00" in raw[:4096]: + return True + try: + raw[:4096].decode("utf-8") + except UnicodeDecodeError: + return True + return False + + +def _iso_from_timestamp(value: float) -> str: + return datetime.fromtimestamp(value, tz=timezone.utc).isoformat() + + +def _safe_scope(value: str | None) -> str: + raw = (value or "interactive").strip() + allowed = [char if char.isalnum() or char in ("-", "_") else "-" for char in raw] + cleaned = "".join(allowed).strip("-_") + return cleaned or "interactive" + + +class _LimitedReadStream: + def __init__(self, stream: object, *, max_bytes: int | None = None) -> None: + self.stream = stream + self.max_bytes = max_bytes + self.bytes_read = 0 + + def read(self, size: int = -1) -> bytes: + chunk = self.stream.read(size) # type: ignore[attr-defined] + if not chunk: + return b"" + self.bytes_read += len(chunk) + if self.max_bytes is not None and self.bytes_read > self.max_bytes: + raise UserFileSizeError(_size_error(self.max_bytes)) + return chunk + + +def _size_error(max_bytes: int) -> str: + return f"File too large (max {_human_size(max_bytes)})" + + +def _human_size(size: int) -> str: + units = ("B", "KB", "MB", "GB", "TB") + value = float(size) + for unit in units: + if value < 1024 or unit == units[-1]: + return f"{value:.0f}{unit}" if unit == "B" else f"{value:.1f}{unit}" + value /= 1024 + return f"{size}B" diff --git a/app-instance/backend/beaver/tools/base.py b/app-instance/backend/beaver/tools/base.py index bc4f1ed..86d05c2 100644 --- a/app-instance/backend/beaver/tools/base.py +++ b/app-instance/backend/beaver/tools/base.py @@ -180,8 +180,10 @@ class ObjectBackedTool(BaseTool): if "current_session_id" not in arguments and hasattr(self.backend, "current_session_id"): arguments["current_session_id"] = context.session_id - if "workspace" not in arguments and hasattr(self.backend, "workspace"): + if "workspace" not in arguments and (hasattr(self.backend, "workspace") or self._backend_accepts_argument("workspace")): arguments["workspace"] = context.workspace + if "services" not in arguments and self._backend_accepts_argument("services"): + arguments["services"] = context.services if "metadata" not in arguments and self._backend_accepts_argument("metadata"): arguments["metadata"] = context.metadata diff --git a/app-instance/backend/beaver/tools/builtins/__init__.py b/app-instance/backend/beaver/tools/builtins/__init__.py index 8afd195..951b61e 100644 --- a/app-instance/backend/beaver/tools/builtins/__init__.py +++ b/app-instance/backend/beaver/tools/builtins/__init__.py @@ -9,6 +9,15 @@ from .skill_view import SkillViewTool, skill_view from .session_search import SessionSearchTool, session_search from .terminal import ExecuteCodeTool, ProcessTool, TerminalTool from .utility import ClarifyTool, DelegateTool, SendMessageTool, SpawnTool, TodoTool +from .user_files import ( + UserFilesCopyToWorkspaceTool, + UserFilesDeleteTool, + UserFilesListTool, + UserFilesMkdirTool, + UserFilesPublishOutputTool, + UserFilesReadTool, + UserFilesWriteTool, +) from .web import WebFetchTool, WebSearchTool __all__ = [ @@ -30,6 +39,13 @@ __all__ = [ "SessionSearchTool", "TerminalTool", "TodoTool", + "UserFilesCopyToWorkspaceTool", + "UserFilesDeleteTool", + "UserFilesListTool", + "UserFilesMkdirTool", + "UserFilesPublishOutputTool", + "UserFilesReadTool", + "UserFilesWriteTool", "ClarifyTool", "WebFetchTool", "WebSearchTool", diff --git a/app-instance/backend/beaver/tools/builtins/filesystem.py b/app-instance/backend/beaver/tools/builtins/filesystem.py index 0603457..0f006ab 100644 --- a/app-instance/backend/beaver/tools/builtins/filesystem.py +++ b/app-instance/backend/beaver/tools/builtins/filesystem.py @@ -14,7 +14,7 @@ from __future__ import annotations from dataclasses import dataclass, field import json -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Iterable @@ -24,6 +24,7 @@ MAX_READ_CHARS = 120_000 MAX_SEARCH_RESULTS = 200 MAX_SEARCH_FILE_BYTES = 2_000_000 MAX_SEARCH_FILES = 5_000 +USER_FILE_VIRTUAL_ROOTS = {"uploads", "outputs", "shared", "tasks"} SKIP_DIR_NAMES = { ".git", ".hg", @@ -161,9 +162,28 @@ def _workspace_root(workspace: str | None) -> Path: return root +def _virtual_user_file_error(user_path: str | None) -> str | None: + raw = str(user_path or ".").replace("\\", "/").strip() + if not raw or raw in {".", "./"}: + return None + try: + parts = [part for part in PurePosixPath(raw.strip("/")).parts if part not in ("", ".")] + except TypeError: + return None + if parts and parts[0] in USER_FILE_VIRTUAL_ROOTS: + return ( + f"{user_path} is a personal agent file system path, not a workspace path. " + "Use user_files_read or user_files_copy_to_workspace for reads; use " + "user_files_write for shared/tasks files or user_files_publish_output for outputs." + ) + return None + + def _resolve_existing_path(workspace: str | None, user_path: str | None) -> tuple[Path, Path]: """Resolve a user path and ensure the real target stays inside workspace.""" + if error := _virtual_user_file_error(user_path): + raise WorkspacePathError(error) root = _workspace_root(workspace) raw_path = Path(user_path or ".").expanduser() candidate = raw_path if raw_path.is_absolute() else root / raw_path @@ -178,6 +198,8 @@ def _resolve_existing_path(workspace: str | None, user_path: str | None) -> tupl def _resolve_writable_path(workspace: str | None, user_path: str | None) -> tuple[Path, Path]: + if error := _virtual_user_file_error(user_path): + raise WorkspacePathError(error) root = _workspace_root(workspace) if not user_path or not str(user_path).strip(): raise WorkspacePathError("path is required") diff --git a/app-instance/backend/beaver/tools/builtins/user_files.py b/app-instance/backend/beaver/tools/builtins/user_files.py new file mode 100644 index 0000000..6a9ea3c --- /dev/null +++ b/app-instance/backend/beaver/tools/builtins/user_files.py @@ -0,0 +1,389 @@ +"""Agent-facing tools for the user-visible file system.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import json +import mimetypes +from pathlib import Path +from typing import Any + +from beaver.foundation.config.loader import load_config +from beaver.services.user_file_resolver import UserFileStorageResolver, build_file_auth_context +from beaver.services.user_files import AgentUserFilePolicy, UserFileError, UserFilePathError, UserFileService + + +MAX_WORKSPACE_STAGE_BYTES = 50 * 1024 * 1024 + + +def _json_result(success: bool, **payload: Any) -> str: + return json.dumps({"success": success, **payload}, ensure_ascii=False, indent=2) + + +async def _service(workspace: str | None, services: dict[str, Any] | None = None) -> UserFileService: + if not workspace: + raise UserFileError("workspace is not configured for user file tools") + config = (services or {}).get("beaver_config") + if config is None: + config = load_config(workspace=workspace) + backend_id = config.backend_identity.backend_id.strip() or config.backend_identity.client_id.strip() or "agent" + auth_context = build_file_auth_context( + username=backend_id, + config=config, + user_id=(services or {}).get("user_id"), + auth_source="beaver-agent-runtime", + ) + return await UserFileStorageResolver( + config=config, + workspace=Path(workspace), + auth_context=auth_context, + ).service() + + +def _agent_policy(services: dict[str, Any] | None = None, metadata: dict[str, Any] | None = None) -> AgentUserFilePolicy: + payload = services or {} + meta = metadata or {} + task_id = str(payload.get("task_id") or meta.get("task_id") or "").strip() or None + fallback = str(payload.get("run_id") or meta.get("run_id") or meta.get("session_id") or "interactive") + return AgentUserFilePolicy(task_id=task_id, fallback_scope=fallback) + + +def _workspace_root(workspace: str | None) -> Path: + if not workspace: + raise UserFilePathError("workspace is not configured for user file tools") + root = Path(workspace).expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + return root + + +def _resolve_workspace_source(workspace: str | None, source_path: str) -> tuple[Path, Path]: + root = _workspace_root(workspace) + if not source_path or not str(source_path).strip(): + raise UserFilePathError("source_path is required") + raw = Path(str(source_path)).expanduser() + candidate = raw if raw.is_absolute() else root / raw + resolved = candidate.resolve(strict=True) + try: + resolved.relative_to(root) + except ValueError as exc: + raise UserFilePathError("source_path escapes workspace") from exc + if not resolved.is_file(): + raise UserFilePathError("source_path must be a file") + return root, resolved + + +def _resolve_workspace_destination(workspace: str | None, target_path: str) -> tuple[Path, Path]: + root = _workspace_root(workspace) + if not target_path or not str(target_path).strip(): + raise UserFilePathError("workspace_path is required") + raw = Path(str(target_path)).expanduser() + if raw.is_absolute(): + raise UserFilePathError("workspace_path must be relative") + candidate = (root / raw).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise UserFilePathError("workspace_path escapes workspace") from exc + return root, candidate + + +def _relative_path(root: Path, path: Path) -> str: + return path.relative_to(root).as_posix() + + +USER_FILES_LIST_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "path": { + "type": "string", + "default": "", + "description": "User file path under uploads, outputs, shared, or tasks. Empty path lists the virtual roots.", + } + }, +} + +USER_FILES_READ_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "path": {"type": "string", "description": "User file path to read."}, + "max_bytes": { + "type": "integer", + "default": 120000, + "minimum": 1, + "maximum": 1000000, + "description": "Maximum bytes to return in model context.", + }, + }, + "required": ["path"], +} + +USER_FILES_WRITE_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "path": {"type": "string", "description": "User file path to create or replace."}, + "content": {"type": "string", "description": "Text content to write."}, + "content_type": {"type": "string", "default": "text/plain"}, + }, + "required": ["path", "content"], +} + +USER_FILES_DELETE_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": {"path": {"type": "string", "description": "User file or directory path to delete."}}, + "required": ["path"], +} + +USER_FILES_MKDIR_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": {"path": {"type": "string", "description": "User file directory path to create."}}, + "required": ["path"], +} + +USER_FILES_COPY_TO_WORKSPACE_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Readable user file path under uploads, outputs, shared, or an authorized tasks namespace.", + }, + "workspace_path": { + "type": "string", + "description": "Optional relative workspace destination. Defaults to user-files/tasks/{task_id}/ or user-files/runs//.", + }, + }, + "required": ["path"], +} + +USER_FILES_PUBLISH_OUTPUT_PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "source_path": { + "type": "string", + "description": "Workspace file path to publish. Absolute paths are allowed only if they stay inside the workspace.", + }, + "target_path": { + "type": "string", + "description": "Output path under outputs/, such as outputs/report.md.", + }, + "content_type": { + "type": "string", + "description": "Optional content type. If omitted, Beaver guesses from the target filename.", + }, + }, + "required": ["source_path", "target_path"], +} + + +@dataclass(slots=True) +class UserFilesListTool: + name: str = "user_files_list" + description: str = ( + "List files and folders in the personal agent file system. Use the virtual roots only: " + "uploads for files the user provides to the agent, outputs for agent-generated results, " + "shared for reusable user/agent reference material, and tasks for files bound to a specific task. " + "An empty path lists the four roots; this tool never exposes MinIO buckets, credentials, or internal workspace paths." + ) + toolset: str = "user_files" + always_available: bool = True + parameters: dict[str, Any] = field(default_factory=lambda: dict(USER_FILES_LIST_PARAMETERS)) + + async def execute(self, *, path: str = "", workspace: str | None = None, services: dict[str, Any] | None = None) -> str: + try: + return _json_result(True, **await (await _service(workspace, services)).browse(path)) + except UserFileError as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class UserFilesReadTool: + name: str = "user_files_read" + description: str = ( + "Read a bounded text preview from the personal agent file system. Use this to inspect user-provided " + "files in uploads, long-lived shared material in shared, task files in tasks, or generated outputs in outputs. " + "The path must stay under uploads, outputs, shared, or tasks; internal workspace and MinIO implementation paths are hidden." + ) + toolset: str = "user_files" + always_available: bool = True + parameters: dict[str, Any] = field(default_factory=lambda: dict(USER_FILES_READ_PARAMETERS)) + + async def execute( + self, + *, + path: str, + max_bytes: int = 120000, + workspace: str | None = None, + services: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + ) -> str: + try: + path = _agent_policy(services, metadata).validate_read(path) + limit = max(1, min(int(max_bytes), 1_000_000)) + return _json_result(True, **await (await _service(workspace, services)).preview(path, max_bytes=limit)) + except UserFileError as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class UserFilesWriteTool: + name: str = "user_files_write" + description: str = ( + "Create or replace a text file in the personal agent file system. Store agent-generated deliverables " + "under outputs, reusable long-lived context under shared, and task-bound files under the current " + "tasks/{task_id}/ namespace. Never write to uploads; uploaded files are immutable agent inputs. " + "For modifications to uploaded files, copy them to the workspace, edit there, then publish to outputs." + ) + toolset: str = "user_files" + always_available: bool = False + parameters: dict[str, Any] = field(default_factory=lambda: dict(USER_FILES_WRITE_PARAMETERS)) + + async def execute( + self, + *, + path: str, + content: str, + content_type: str = "text/plain", + workspace: str | None = None, + services: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + ) -> str: + try: + path = _agent_policy(services, metadata).validate_write(path) + return _json_result(True, **await (await _service(workspace, services)).write_file(path, content, content_type=content_type)) + except UserFileError as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class UserFilesDeleteTool: + name: str = "user_files_delete" + description: str = ( + "Agent deletion is disabled for the personal agent file system. User-visible file deletion is owned by " + "the Files page or user-side APIs; agents should use task/workspace cleanup instead." + ) + toolset: str = "user_files" + always_available: bool = False + parameters: dict[str, Any] = field(default_factory=lambda: dict(USER_FILES_DELETE_PARAMETERS)) + + async def execute(self, *, path: str, workspace: str | None = None, services: dict[str, Any] | None = None) -> str: + try: + _agent_policy(services).validate_delete(path) + return _json_result(False, path=path, deleted=False) + except UserFileError as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class UserFilesMkdirTool: + name: str = "user_files_mkdir" + description: str = ( + "Create a subfolder in the personal agent file system under uploads, outputs, shared, or tasks. " + "Use folders to organize agent outputs, reusable shared material, or current task-specific files. " + "Do not create folders under uploads because uploads is user-owned input storage." + ) + toolset: str = "user_files" + always_available: bool = False + parameters: dict[str, Any] = field(default_factory=lambda: dict(USER_FILES_MKDIR_PARAMETERS)) + + async def execute( + self, + *, + path: str, + workspace: str | None = None, + services: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + ) -> str: + try: + path = _agent_policy(services, metadata).validate_mkdir(path) + return _json_result(True, **await (await _service(workspace, services)).mkdir(path)) + except UserFileError as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class UserFilesCopyToWorkspaceTool: + name: str = "user_files_copy_to_workspace" + description: str = ( + "Copy a readable file from the personal agent file system into the internal workspace before editing, " + "running, or validating it. Use this for user-uploaded files under uploads: the original upload remains " + "unchanged, and the returned workspace_path can be used with workspace tools like read_file or patch_file." + ) + toolset: str = "user_files" + always_available: bool = False + parameters: dict[str, Any] = field(default_factory=lambda: dict(USER_FILES_COPY_TO_WORKSPACE_PARAMETERS)) + + async def execute( + self, + *, + path: str, + workspace_path: str | None = None, + workspace: str | None = None, + services: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + ) -> str: + try: + policy = _agent_policy(services, metadata) + path = policy.validate_read(path) + content = await (await _service(workspace, services)).download(path) + if content.size > MAX_WORKSPACE_STAGE_BYTES: + raise UserFilePathError(f"File is too large to copy to workspace (max {MAX_WORKSPACE_STAGE_BYTES} bytes)") + default_path = f"user-files/{policy.task_namespace}/{Path(path).name}" + root, destination = _resolve_workspace_destination(workspace, workspace_path or default_path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(content.content) + return _json_result( + True, + path=path, + workspace_path=_relative_path(root, destination), + bytes=len(content.content), + content_type=content.content_type, + ) + except UserFileError as exc: + return _json_result(False, error=str(exc), path=path) + except OSError as exc: + return _json_result(False, error=str(exc), path=path) + + +@dataclass(slots=True) +class UserFilesPublishOutputTool: + name: str = "user_files_publish_output" + description: str = ( + "Publish a validated workspace file to the personal agent file system under outputs/. Use this after " + "staging and editing files in the workspace. Publishing never writes to uploads, and it hides MinIO " + "bucket, namespace, and credential details from the agent." + ) + toolset: str = "user_files" + always_available: bool = False + parameters: dict[str, Any] = field(default_factory=lambda: dict(USER_FILES_PUBLISH_OUTPUT_PARAMETERS)) + + async def execute( + self, + *, + source_path: str, + target_path: str, + content_type: str | None = None, + workspace: str | None = None, + services: dict[str, Any] | None = None, + ) -> str: + try: + root, source = _resolve_workspace_source(workspace, source_path) + normalized_target = target_path.strip().strip("/") + if not normalized_target.startswith("outputs/"): + raise UserFilePathError("Published output target must be under outputs/") + guessed_type, _ = mimetypes.guess_type(normalized_target) + raw = source.read_bytes() + entry = await (await _service(workspace, services)).write_file( + normalized_target, + raw, + content_type=content_type or guessed_type or "application/octet-stream", + ) + return _json_result( + True, + source_path=_relative_path(root, source), + target_path=normalized_target, + bytes=len(raw), + **entry, + ) + except UserFileError as exc: + return _json_result(False, error=str(exc), source_path=source_path, target_path=target_path) + except OSError as exc: + return _json_result(False, error=str(exc), source_path=source_path, target_path=target_path) diff --git a/app-instance/backend/docs/security/user-filesystem-minio-authz.md b/app-instance/backend/docs/security/user-filesystem-minio-authz.md new file mode 100644 index 0000000..affe1fd --- /dev/null +++ b/app-instance/backend/docs/security/user-filesystem-minio-authz.md @@ -0,0 +1,104 @@ +# User File System MinIO/AuthZ Setup + +The user file system is exposed through Beaver APIs and `user_files_*` tools. MinIO remains an implementation detail. + +The ordinary Files page should only call Beaver's `/api/user-files/*` routes and render the virtual roots `uploads/`, `outputs/`, `shared/`, and `tasks/`. It should not show bucket names, endpoint fields, access keys, secret keys, object prefixes, or MinIO administration actions. + +## AuthZ Settings + +Each backend identity can store MinIO settings in AuthZ: + +```bash +curl -X POST "$AUTHZ_URL/backends/$BACKEND_ID/settings/minio" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $AUTHZ_ADMIN_TOKEN" \ + -d '{ + "endpoint": "minio.example.internal:9000", + "access_key": "user-access-key", + "secret_key": "user-secret-key", + "bucket": "beaver-user-files", + "namespace": "users/{backend_id}", + "secure": false, + "region": null + }' +``` + +Public reads return masked settings. Internal reads require `AUTHZ_INTERNAL_TOKEN` and return the secret key for protected MCP services. + +Deployed personal files use a shared bucket with a backend-scoped namespace. For backend `alice`, Beaver maps: + +- `uploads/report.pdf` to `users/alice/uploads/report.pdf` +- `outputs/summary.md` to `users/alice/outputs/summary.md` +- `tasks/task-123/result.json` to `users/alice/tasks/task-123/result.json` + +The MinIO policy for Alice's access key must be limited to `beaver-user-files/users/alice/*`. The frontend must still only show Beaver virtual paths, not the shared bucket or namespace. + +Check the public, masked view: + +```bash +curl "$AUTHZ_URL/backends/$BACKEND_ID/settings/minio" \ + -H "Authorization: Bearer $AUTHZ_ADMIN_TOKEN" +``` + +Check the internal protected view used by MCP services: + +```bash +curl "$AUTHZ_URL/internal/backends/$BACKEND_ID/settings/minio" \ + -H "Authorization: Bearer $AUTHZ_INTERNAL_TOKEN" +``` + +## Protected MinIO MCP + +Run the MinIO MCP service in protected mode: + +```bash +bw-minio-mcp serve \ + --host 0.0.0.0 \ + --port 8001 \ + --authz-url "$AUTHZ_URL" \ + --authz-token "$AUTHZ_INTERNAL_TOKEN" \ + --resource-server-url "$MINIO_MCP_PUBLIC_URL/mcp" \ + --state-root /var/lib/bw-minio-mcp +``` + +In protected mode, the MCP service does not use static MinIO credentials at startup. Each authenticated tool call resolves the backend identity from the bearer token, loads that backend's MinIO settings from AuthZ, and constructs a per-call provider. + +Outside protected mode, `bw-minio-mcp serve` requires explicit `--endpoint`, `--access-key`, and `--secret-key` values. It intentionally has no embedded production fallback credentials. + +## Beaver Runtime + +Beaver should register the MinIO MCP endpoint with backend-token auth when raw object tools are needed: + +```json +{ + "tools": { + "mcpServers": { + "minio_mcp": { + "url": "https://minio-mcp.example.internal/mcp", + "auth": "oauth_backend_token", + "authAudience": "mcp:minio_mcp" + } + } + }, + "authz": { + "baseUrl": "https://authz.example.internal", + "backendId": "backend-user-id" + } +} +``` + +Product-level file interactions should still go through Beaver's user file system: + +- Frontend: `/api/user-files/status`, `/api/user-files/browse`, `/api/user-files/upload`, `/api/user-files/preview`, `/api/user-files/download`, `/api/user-files/delete`, and `/api/user-files/mkdir`. +- Agent tools: `user_files_list`, `user_files_read`, `user_files_write`, `user_files_delete`, and `user_files_mkdir`. +- Storage boundary: only `uploads/`, `outputs/`, `shared/`, and `tasks/` are valid user paths. + +The local workspace browser APIs and generic filesystem tools are retained for runtime/development compatibility, but they are not the user-visible file boundary. + +## Verification Checklist + +- The Files page root renders exactly `uploads`, `outputs`, `shared`, and `tasks`. +- The Files page source does not call `/api/workspace/browse`. +- `/api/user-files/status` does not return local workspace paths or MinIO bucket details. +- AuthZ public settings responses mask `secret_key`. +- Protected `BW_MinIO_Mcp` returns a clear configuration error if a backend has no MinIO settings instead of falling back to another user's credentials. diff --git a/app-instance/backend/docs/security/user-filesystem-tooling.md b/app-instance/backend/docs/security/user-filesystem-tooling.md new file mode 100644 index 0000000..453707c --- /dev/null +++ b/app-instance/backend/docs/security/user-filesystem-tooling.md @@ -0,0 +1,12 @@ +# User File System Tooling Boundary + +The `personal-user-filesystem` change adds `user_files_*` tools for files that users can upload, inspect, and receive from agents. These tools enforce the same virtual roots as the web API: + +- `uploads/` +- `outputs/` +- `shared/` +- `tasks/` + +The existing local workspace filesystem tools remain registered for internal runtime and development workflows. They are workspace-scoped, but they are not the user-visible file boundary. Agents should use `user_files_*` tools when reading user-provided files or writing user-facing outputs. + +Follow-up for stronger isolation: add a runtime policy switch that disables or narrows local workspace filesystem tools for ordinary personal-agent tasks, while keeping `user_files_*` available. diff --git a/app-instance/backend/pyproject.toml b/app-instance/backend/pyproject.toml index 4abada7..369f6de 100644 --- a/app-instance/backend/pyproject.toml +++ b/app-instance/backend/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "httpx>=0.28.0,<1.0.0", "json-repair>=0.39.0,<1.0.0", "litellm>=1.79.0,<2.0.0", + "minio>=7.2.0,<8.0.0", "openai>=1.79.0,<2.0.0", "pydantic>=2.12.0,<3.0.0", "python-multipart>=0.0.20,<1.0.0", diff --git a/app-instance/backend/tests/unit/test_config_loader.py b/app-instance/backend/tests/unit/test_config_loader.py index 622660b..6705a26 100644 --- a/app-instance/backend/tests/unit/test_config_loader.py +++ b/app-instance/backend/tests/unit/test_config_loader.py @@ -199,4 +199,5 @@ def test_load_config_adds_managed_local_mcp_servers(tmp_path) -> None: assert local.kind == "local" assert local.category == "filesystem" assert local.managed is True + assert local.display_name == "个人智能体文件系统工具" assert "beaver.interfaces.mcp.tools_server" in local.args diff --git a/app-instance/backend/tests/unit/test_filesystem_tools.py b/app-instance/backend/tests/unit/test_filesystem_tools.py index 3199365..0009c5d 100644 --- a/app-instance/backend/tests/unit/test_filesystem_tools.py +++ b/app-instance/backend/tests/unit/test_filesystem_tools.py @@ -6,7 +6,7 @@ import os from pathlib import Path from beaver.tools import ObjectBackedTool, ToolContext -from beaver.tools.builtins import ListDirectoryTool, ReadFileTool, SearchFilesTool +from beaver.tools.builtins import ListDirectoryTool, PatchFileTool, ReadFileTool, SearchFilesTool, WriteFileTool def _run_tool(tool, arguments: dict, workspace: Path): @@ -127,3 +127,23 @@ def test_read_file_rejects_binary_files(tmp_path: Path) -> None: assert payload["success"] is False assert "binary" in payload["error"] + +def test_workspace_tools_reject_user_file_virtual_paths(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + + read = _run_tool(ReadFileTool(), {"path": "uploads/get_helm.sh"}, workspace) + listed = _run_tool(ListDirectoryTool(), {"path": "outputs"}, workspace) + written = _run_tool(WriteFileTool(), {"path": "shared/profile.json", "content": "{}"}, workspace) + patched = _run_tool( + PatchFileTool(), + {"path": "tasks/task-123/draft.md", "old_text": "a", "new_text": "b"}, + workspace, + ) + + for result in (read, listed, written, patched): + payload = _payload(result) + assert result.success is False + assert payload["success"] is False + assert "personal agent file system path" in payload["error"] + assert "user_files_read" in payload["error"] diff --git a/app-instance/backend/tests/unit/test_mcp_tools_server.py b/app-instance/backend/tests/unit/test_mcp_tools_server.py new file mode 100644 index 0000000..52a0b24 --- /dev/null +++ b/app-instance/backend/tests/unit/test_mcp_tools_server.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from beaver.interfaces.mcp.tools_server import _category_tools + + +def test_local_filesystem_mcp_exposes_personal_user_file_tools_only(tmp_path) -> None: + tools, _context = _category_tools("filesystem", tmp_path) + + names = [tool.spec.name for tool in tools] + + assert names == [ + "user_files_list", + "user_files_read", + "user_files_write", + "user_files_mkdir", + "user_files_copy_to_workspace", + "user_files_publish_output", + ] + assert "read_file" not in names + assert "search_files" not in names + assert "list_directory" not in names + assert all("personal agent file system" in tool.spec.description for tool in tools) diff --git a/app-instance/backend/tests/unit/test_user_file_service.py b/app-instance/backend/tests/unit/test_user_file_service.py new file mode 100644 index 0000000..a1fcf53 --- /dev/null +++ b/app-instance/backend/tests/unit/test_user_file_service.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from io import BytesIO + +import pytest + +from beaver.services.user_files import ( + LocalUserFileStorage, + MinIOStorageConfig, + MinIOUserFileStorage, + UserFileNotFoundError, + UserFilePathError, + UserFileSizeError, + UserFileService, + normalize_user_path, +) + + +def test_normalize_user_path_accepts_fixed_roots() -> None: + assert normalize_user_path("uploads/readme.txt", allow_root=False) == "uploads/readme.txt" + assert normalize_user_path("outputs/report.md", allow_root=False) == "outputs/report.md" + assert normalize_user_path("tasks/task-123/draft.md", allow_root=False) == "tasks/task-123/draft.md" + assert normalize_user_path("", allow_root=True) == "" + + +@pytest.mark.parametrize( + "path", + [ + "../secret.txt", + "/uploads/input.txt", + "/outputs/result.txt", + "/shared/profile.json", + "/tasks/task-123/draft.md", + "uploads/../state/config.json", + "memory/private.txt", + "uploads/.internal", + "", + ], +) +def test_normalize_user_path_rejects_invalid_paths(path: str) -> None: + with pytest.raises(UserFilePathError): + normalize_user_path(path, allow_root=False) + + +@pytest.mark.asyncio +async def test_user_file_service_root_and_round_trip(tmp_path) -> None: + service = UserFileService(LocalUserFileStorage(tmp_path / "user-files")) + + root = await service.browse("") + uploaded = await service.upload( + "uploads", + "hello.txt", + b"hello user files", + content_type="text/plain", + ) + uploads = await service.browse("uploads") + preview = await service.preview("uploads/hello.txt") + downloaded = await service.download("uploads/hello.txt") + deleted = await service.delete("uploads/hello.txt") + + assert [item["name"] for item in root["items"]] == ["uploads", "outputs", "shared", "tasks"] + assert uploaded["path"] == "uploads/hello.txt" + assert uploaded["content_type"] == "text/plain" + assert [item["name"] for item in uploads["items"]] == ["hello.txt"] + assert preview["content"] == "hello user files" + assert downloaded.content == b"hello user files" + assert deleted is True + + +@pytest.mark.asyncio +async def test_user_file_service_stream_upload_and_size_limit(tmp_path) -> None: + service = UserFileService(LocalUserFileStorage(tmp_path / "user-files")) + + uploaded = await service.upload_stream( + "uploads", + "streamed.txt", + BytesIO(b"streamed user file"), + content_type="text/plain", + max_bytes=1024, + part_size=4, + ) + preview = await service.preview("uploads/streamed.txt") + + assert uploaded["path"] == "uploads/streamed.txt" + assert uploaded["size"] == len(b"streamed user file") + assert preview["content"] == "streamed user file" + + with pytest.raises(UserFileSizeError): + await service.upload_stream( + "uploads", + "too-large.txt", + BytesIO(b"abcdef"), + content_type="text/plain", + max_bytes=5, + part_size=2, + ) + with pytest.raises(UserFileNotFoundError): + await service.preview("uploads/too-large.txt") + + +@pytest.mark.asyncio +async def test_user_file_service_rejects_root_delete_and_traversal(tmp_path) -> None: + service = UserFileService(LocalUserFileStorage(tmp_path / "user-files")) + + with pytest.raises(UserFilePathError): + await service.delete("uploads") + + with pytest.raises(UserFilePathError): + await service.upload("../workspace", "hello.txt", b"x", content_type="text/plain") + + +@pytest.mark.asyncio +async def test_user_file_service_creates_nested_directories(tmp_path) -> None: + service = UserFileService(LocalUserFileStorage(tmp_path / "user-files")) + + created = await service.mkdir("tasks/task-123/references") + tasks = await service.browse("tasks/task-123") + + assert created["path"] == "tasks/task-123/references" + assert created["type"] == "directory" + assert [item["name"] for item in tasks["items"]] == ["references"] + + +def test_minio_storage_maps_virtual_paths_under_namespace() -> None: + storage = object.__new__(MinIOUserFileStorage) + storage.config = MinIOStorageConfig( + endpoint="minio.local:9000", + access_key="alice-access", + secret_key="alice-secret", + bucket="beaver-user-files", + namespace="users/alice", + ) + + assert storage._object_name("uploads/report.pdf") == "users/alice/uploads/report.pdf" + assert storage._object_name("tasks/task-123/result.json") == "users/alice/tasks/task-123/result.json" + assert storage._user_path("users/alice/outputs/summary.md") == "outputs/summary.md" + + +def test_minio_storage_rejects_paths_that_escape_namespace() -> None: + storage = object.__new__(MinIOUserFileStorage) + storage.config = MinIOStorageConfig( + endpoint="minio.local:9000", + access_key="alice-access", + secret_key="alice-secret", + bucket="beaver-user-files", + namespace="users/alice", + ) + + with pytest.raises(UserFilePathError): + storage._object_name("uploads/../state/config.json") + + with pytest.raises(UserFilePathError): + storage._user_path("users/bob/uploads/secret.txt") diff --git a/app-instance/backend/tests/unit/test_user_file_tools.py b/app-instance/backend/tests/unit/test_user_file_tools.py new file mode 100644 index 0000000..d180251 --- /dev/null +++ b/app-instance/backend/tests/unit/test_user_file_tools.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import json + +import pytest + +from beaver.foundation.config.schema import AuthzConfig, BackendIdentityConfig, BeaverConfig +from beaver.tools.base import ObjectBackedTool, ToolContext +from beaver.tools.builtins import ( + UserFilesCopyToWorkspaceTool, + UserFilesListTool, + UserFilesPublishOutputTool, + UserFilesReadTool, + UserFilesWriteTool, +) + + +@pytest.mark.asyncio +async def test_user_file_tools_write_read_and_list(tmp_path) -> None: + context = ToolContext(workspace=str(tmp_path)) + write = ObjectBackedTool(UserFilesWriteTool()) + read = ObjectBackedTool(UserFilesReadTool()) + list_files = ObjectBackedTool(UserFilesListTool()) + + written = await write.invoke( + {"path": "outputs/summary.md", "content": "# Summary", "content_type": "text/markdown"}, + context, + ) + listed = await list_files.invoke({"path": "outputs"}, context) + loaded = await read.invoke({"path": "outputs/summary.md"}, context) + + assert written.success is True + assert json.loads(written.content)["path"] == "outputs/summary.md" + assert listed.success is True + assert [item["name"] for item in json.loads(listed.content)["items"]] == ["summary.md"] + assert loaded.success is True + assert json.loads(loaded.content)["content"] == "# Summary" + + +@pytest.mark.asyncio +async def test_user_file_tools_reject_agent_write_to_uploads(tmp_path) -> None: + context = ToolContext(workspace=str(tmp_path)) + write = ObjectBackedTool(UserFilesWriteTool()) + + result = await write.invoke({"path": "uploads/notes.txt", "content": "changed"}, context) + + assert result.success is False + assert "uploads/ is user-provided input storage" in (result.error or "") + + +@pytest.mark.asyncio +async def test_user_file_tools_enforce_current_task_namespace(tmp_path) -> None: + context = ToolContext(workspace=str(tmp_path), services={"task_id": "task-123"}) + write = ObjectBackedTool(UserFilesWriteTool()) + + current = await write.invoke({"path": "tasks/task-123/drafts/notes.md", "content": "ok"}, context) + direct = await write.invoke({"path": "tasks/notes.md", "content": "bad"}, context) + other = await write.invoke({"path": "tasks/task-456/notes.md", "content": "bad"}, context) + + assert current.success is True + assert direct.success is False + assert "tasks/task-123/" in (direct.error or "") + assert other.success is False + assert "tasks/task-123/" in (other.error or "") + + +@pytest.mark.asyncio +async def test_user_file_tools_allow_shared_context_write(tmp_path) -> None: + context = ToolContext(workspace=str(tmp_path), services={"task_id": "task-123"}) + write = ObjectBackedTool(UserFilesWriteTool()) + read = ObjectBackedTool(UserFilesReadTool()) + + written = await write.invoke({"path": "shared/profile.json", "content": "{\"name\":\"Alice\"}"}, context) + loaded = await read.invoke({"path": "shared/profile.json"}, context) + + assert written.success is True + assert loaded.success is True + assert json.loads(loaded.content)["content"] == "{\"name\":\"Alice\"}" + + +@pytest.mark.asyncio +async def test_user_file_tools_copy_to_workspace_and_publish_output(tmp_path) -> None: + uploads_dir = tmp_path / "user_files" / "uploads" + uploads_dir.mkdir(parents=True) + (uploads_dir / "get_helm.sh").write_text(": ${USE_SUDO:=\"true\"}\n", encoding="utf-8") + context = ToolContext( + workspace=str(tmp_path), + services={"task_id": "task-123"}, + metadata={"run_id": "run-1"}, + ) + copy_tool = ObjectBackedTool(UserFilesCopyToWorkspaceTool()) + publish_tool = ObjectBackedTool(UserFilesPublishOutputTool()) + read = ObjectBackedTool(UserFilesReadTool()) + + copied = await copy_tool.invoke({"path": "uploads/get_helm.sh"}, context) + copied_payload = json.loads(copied.content) + staged = tmp_path / copied_payload["workspace_path"] + staged.write_text(": ${USE_SUDO:=\"false\"}\n", encoding="utf-8") + published = await publish_tool.invoke( + {"source_path": copied_payload["workspace_path"], "target_path": "outputs/get_helm.no-sudo.sh"}, + context, + ) + original = await read.invoke({"path": "uploads/get_helm.sh"}, context) + output = await read.invoke({"path": "outputs/get_helm.no-sudo.sh"}, context) + + assert copied.success is True + assert copied_payload["workspace_path"] == "user-files/tasks/task-123/get_helm.sh" + assert published.success is True + assert json.loads(original.content)["content"] == ": ${USE_SUDO:=\"true\"}\n" + assert json.loads(output.content)["content"] == ": ${USE_SUDO:=\"false\"}\n" + + +@pytest.mark.asyncio +async def test_user_file_publish_rejects_non_output_target_and_workspace_escape(tmp_path) -> None: + context = ToolContext(workspace=str(tmp_path)) + source = tmp_path / "result.txt" + source.write_text("done", encoding="utf-8") + outside = tmp_path.parent / "outside.txt" + outside.write_text("outside", encoding="utf-8") + publish_tool = ObjectBackedTool(UserFilesPublishOutputTool()) + + upload_target = await publish_tool.invoke({"source_path": "result.txt", "target_path": "uploads/result.txt"}, context) + escaped_source = await publish_tool.invoke({"source_path": str(outside), "target_path": "outputs/result.txt"}, context) + + assert upload_target.success is False + assert "outputs/" in (upload_target.error or "") + assert escaped_source.success is False + assert "escapes workspace" in (escaped_source.error or "") + + +@pytest.mark.asyncio +async def test_user_file_tools_reject_internal_workspace_paths(tmp_path) -> None: + context = ToolContext(workspace=str(tmp_path)) + read = ObjectBackedTool(UserFilesReadTool()) + write = ObjectBackedTool(UserFilesWriteTool()) + + read_result = await read.invoke({"path": "uploads/../../state/secrets.json"}, context) + write_result = await write.invoke({"path": "workspace/debug.txt", "content": "x"}, context) + + assert read_result.success is False + assert "Parent-directory traversal" in read_result.error + assert write_result.success is False + assert "Path must be under" in write_result.error + + +@pytest.mark.asyncio +async def test_user_file_tools_reject_absolute_style_user_paths(tmp_path) -> None: + context = ToolContext(workspace=str(tmp_path), services={"task_id": "task-123"}) + read = ObjectBackedTool(UserFilesReadTool()) + write = ObjectBackedTool(UserFilesWriteTool()) + list_files = ObjectBackedTool(UserFilesListTool()) + + read_result = await read.invoke({"path": "/uploads/input.txt"}, context) + write_result = await write.invoke({"path": "/outputs/result.txt", "content": "x"}, context) + task_write = await write.invoke({"path": "/tasks/task-123/draft.md", "content": "x"}, context) + list_result = await list_files.invoke({"path": "/shared/profile.json"}, context) + + for result in (read_result, write_result, task_write, list_result): + assert result.success is False + assert "Absolute paths are not allowed" in (result.error or "") + + +@pytest.mark.asyncio +async def test_user_file_tools_report_missing_deployed_minio_settings(tmp_path, monkeypatch) -> None: + monkeypatch.delenv("BEAVER_AUTHZ_INTERNAL_TOKEN", raising=False) + monkeypatch.delenv("AUTHZ_INTERNAL_TOKEN", raising=False) + config = BeaverConfig( + authz=AuthzConfig(enabled=True, base_url="http://authz.local"), + backend_identity=BackendIdentityConfig(backend_id="alice", client_id="alice", client_secret="secret"), + ) + context = ToolContext(workspace=str(tmp_path), services={"beaver_config": config}) + write = ObjectBackedTool(UserFilesWriteTool()) + + result = await write.invoke({"path": "outputs/summary.md", "content": "# Summary"}, context) + + assert result.success is False + assert "AuthZ internal token is not configured" in (result.error or "") diff --git a/app-instance/backend/tests/unit/test_web_files_api.py b/app-instance/backend/tests/unit/test_web_files_api.py index f40bda4..32fd6f5 100644 --- a/app-instance/backend/tests/unit/test_web_files_api.py +++ b/app-instance/backend/tests/unit/test_web_files_api.py @@ -6,6 +6,14 @@ from fastapi.testclient import TestClient from beaver.interfaces.web.app import create_app from beaver.services.agent_service import AgentService +from beaver.services.user_file_resolver import UserFileStorageResolver +from beaver.services.user_files import LocalUserFileStorage, UserFileService + + +def _auth_headers(app, username: str = "alice") -> dict[str, str]: + token = f"test-token-{username}" + app.state.auth_tokens[token] = username + return {"Authorization": f"Bearer {token}"} def test_workspace_browser_api_manages_workspace_files(tmp_path: Path) -> None: @@ -68,3 +76,145 @@ def test_attachment_file_api_round_trips_uploaded_file(tmp_path: Path) -> None: assert deleted.status_code == 200 assert deleted.json() == {"ok": True} assert missing.status_code == 404 + + +def test_user_files_api_uses_virtual_roots_and_hides_workspace(tmp_path: Path) -> None: + service = AgentService(workspace=tmp_path) + app = create_app(service=service, manage_service_lifecycle=False) + + with TestClient(app) as client: + headers = _auth_headers(app) + root = client.get("/api/user-files/browse", headers=headers) + status = client.get("/api/user-files/status", headers=headers) + upload = client.post( + "/api/user-files/upload", + data={"path": "uploads"}, + files={"file": ("hello.txt", b"hello user files", "text/plain")}, + headers=headers, + ) + uploads = client.get("/api/user-files/browse", params={"path": "uploads"}, headers=headers) + preview = client.get("/api/user-files/preview", params={"path": "uploads/hello.txt"}, headers=headers) + download = client.get("/api/user-files/download", params={"path": "uploads/hello.txt"}, headers=headers) + + assert root.status_code == 200 + assert [item["name"] for item in root.json()["items"]] == ["uploads", "outputs", "shared", "tasks"] + assert all("bucket" not in item for item in root.json()["items"]) + assert status.status_code == 200 + assert status.json()["workspace_visible"] is False + assert "base_path" not in status.json() + assert upload.status_code == 200 + assert upload.json()["path"] == "uploads/hello.txt" + assert uploads.status_code == 200 + assert [item["name"] for item in uploads.json()["items"]] == ["hello.txt"] + assert preview.status_code == 200 + assert preview.json()["content"] == "hello user files" + assert download.status_code == 200 + assert download.content == b"hello user files" + + +def test_user_files_api_rejects_invalid_paths_and_root_delete(tmp_path: Path) -> None: + service = AgentService(workspace=tmp_path) + app = create_app(service=service, manage_service_lifecycle=False) + + with TestClient(app) as client: + headers = _auth_headers(app) + traversal = client.get("/api/user-files/browse", params={"path": "uploads/../state"}, headers=headers) + unknown_root = client.get("/api/user-files/browse", params={"path": "memory/private.txt"}, headers=headers) + absolute_browse = client.get("/api/user-files/browse", params={"path": "/uploads/input.txt"}, headers=headers) + absolute_download = client.get("/api/user-files/download", params={"path": "/outputs/result.txt"}, headers=headers) + absolute_preview = client.get("/api/user-files/preview", params={"path": "/shared/profile.json"}, headers=headers) + absolute_mkdir = client.post("/api/user-files/mkdir", params={"path": "/tasks/task-123/draft.md"}, headers=headers) + absolute_upload = client.post( + "/api/user-files/upload", + data={"path": "/uploads"}, + files={"file": ("input.txt", b"x", "text/plain")}, + headers=headers, + ) + delete_root = client.delete("/api/user-files/delete", params={"path": "uploads"}, headers=headers) + + assert traversal.status_code == 400 + assert unknown_root.status_code == 400 + assert absolute_browse.status_code == 400 + assert absolute_download.status_code == 400 + assert absolute_preview.status_code == 400 + assert absolute_mkdir.status_code == 400 + assert absolute_upload.status_code == 400 + assert delete_root.status_code == 400 + + +def test_user_files_api_rejects_anonymous_access_before_storage(tmp_path: Path) -> None: + service = AgentService(workspace=tmp_path) + app = create_app(service=service, manage_service_lifecycle=False) + + with TestClient(app) as client: + browse = client.get("/api/user-files/browse") + status = client.get("/api/user-files/status") + upload = client.post( + "/api/user-files/upload", + data={"path": "uploads"}, + files={"file": ("hello.txt", b"hello user files", "text/plain")}, + ) + delete = client.delete("/api/user-files/delete", params={"path": "uploads/hello.txt"}) + mkdir = client.post("/api/user-files/mkdir", params={"path": "uploads/new"}) + + assert browse.status_code == 401 + assert status.status_code == 401 + assert upload.status_code == 401 + assert delete.status_code == 401 + assert mkdir.status_code == 401 + + +def test_user_files_api_authenticated_request_resolves_identity(tmp_path: Path, monkeypatch) -> None: + service = AgentService(workspace=tmp_path) + app = create_app(service=service, manage_service_lifecycle=False) + seen = [] + + async def fake_service(self): + seen.append(self.auth_context) + return UserFileService(LocalUserFileStorage(tmp_path / "user-files")) + + monkeypatch.setattr(UserFileStorageResolver, "service", fake_service) + + with TestClient(app) as client: + alice_headers = _auth_headers(app, "alice") + upload = client.post( + "/api/user-files/upload", + data={"path": "uploads"}, + files={"file": ("alice.txt", b"alice", "text/plain")}, + headers=alice_headers, + ) + + assert upload.status_code == 200 + assert seen + assert seen[0].username == "alice" + assert seen[0].backend_id == "alice" + assert seen[0].storage_namespace == "users/alice" + + +def test_user_files_api_streams_upload_and_enforces_configured_limit(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("BEAVER_USER_FILES_MAX_UPLOAD_BYTES", "5") + service = AgentService(workspace=tmp_path) + app = create_app(service=service, manage_service_lifecycle=False) + + with TestClient(app) as client: + headers = _auth_headers(app) + ok_upload = client.post( + "/api/user-files/upload", + data={"path": "uploads"}, + files={"file": ("small.txt", b"abcde", "text/plain")}, + headers=headers, + ) + too_large = client.post( + "/api/user-files/upload", + data={"path": "uploads"}, + files={"file": ("large.txt", b"abcdef", "text/plain")}, + headers=headers, + ) + preview = client.get("/api/user-files/preview", params={"path": "uploads/small.txt"}, headers=headers) + + assert ok_upload.status_code == 200 + assert ok_upload.json()["path"] == "uploads/small.txt" + assert too_large.status_code == 413 + assert "File too large" in too_large.json()["detail"] + assert preview.status_code == 200 + assert preview.json()["content"] == "abcde" diff --git a/app-instance/backend/uv.lock b/app-instance/backend/uv.lock index 43b08d5..86cc5bc 100644 --- a/app-instance/backend/uv.lock +++ b/app-instance/backend/uv.lock @@ -192,6 +192,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/3c0a35f46e52108d4707c44b95cfe2afcafc50800b5450c197454569b776/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:3d3f05610594151994ca9ccb3c771115bdb4daef161976a266f0dd8aa9996b8f", size = 54393, upload-time = "2025-07-30T10:01:40.97Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/98bbd6ee89febd4f212696f13c03ca302b8552e7dbf9c8efa11ea4a388c3/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8b8efee945193e667a396cbc7b4fb7d357297d6234d30a489905d96caabde56b", size = 29328, upload-time = "2025-07-30T10:01:41.916Z" }, + { url = "https://files.pythonhosted.org/packages/43/24/90a01c0ef12ac91a6be05969f29944643bc1e5e461155ae6559befa8f00b/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3c6702abc36bf3ccba3f802b799505def420a1b7039862014a65db3205967f5a", size = 31269, upload-time = "2025-07-30T10:01:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d3/942aa10782b2697eee7af5e12eeff5ebb325ccfb86dd8abda54174e377e4/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44", size = 86558, upload-time = "2025-07-30T10:01:43.943Z" }, + { url = "https://files.pythonhosted.org/packages/0d/82/b484f702fec5536e71836fc2dbc8c5267b3f6e78d2d539b4eaa6f0db8bf8/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb", size = 92364, upload-time = "2025-07-30T10:01:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c1/a606ff83b3f1735f3759ad0f2cd9e038a0ad11a3de3b6c673aa41c24bb7b/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92", size = 85637, upload-time = "2025-07-30T10:01:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/44/b4/678503f12aceb0262f84fa201f6027ed77d71c5019ae03b399b97caa2f19/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85", size = 91934, upload-time = "2025-07-30T10:01:47.203Z" }, + { url = "https://files.pythonhosted.org/packages/f0/c7/f36bd08ef9bd9f0a9cff9428406651f5937ce27b6c5b07b92d41f91ae541/argon2_cffi_bindings-25.1.0-cp314-cp314t-win32.whl", hash = "sha256:84a461d4d84ae1295871329b346a97f68eade8c53b6ed9a7ca2d7467f3c8ff6f", size = 28158, upload-time = "2025-07-30T10:01:48.341Z" }, + { url = "https://files.pythonhosted.org/packages/b3/80/0106a7448abb24a2c467bf7d527fe5413b7fdfa4ad6d6a96a43a62ef3988/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b55aec3565b65f56455eebc9b9f34130440404f27fe21c3b375bf1ea4d8fbae6", size = 32597, upload-time = "2025-07-30T10:01:49.112Z" }, + { url = "https://files.pythonhosted.org/packages/05/b8/d663c9caea07e9180b2cb662772865230715cbd573ba3b5e81793d580316/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:87c33a52407e4c41f3b70a9c2d3f6056d88b10dad7695be708c5021673f55623", size = 28231, upload-time = "2025-07-30T10:01:49.92Z" }, + { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121, upload-time = "2025-07-30T10:01:50.815Z" }, + { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177, upload-time = "2025-07-30T10:01:51.681Z" }, + { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090, upload-time = "2025-07-30T10:01:53.184Z" }, + { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, + { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, + { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, + { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180, upload-time = "2025-07-30T10:01:57.759Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715, upload-time = "2025-07-30T10:01:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149, upload-time = "2025-07-30T10:01:59.329Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -244,6 +287,7 @@ dependencies = [ { name = "httpx" }, { name = "json-repair" }, { name = "litellm" }, + { name = "minio" }, { name = "openai" }, { name = "pydantic" }, { name = "python-multipart" }, @@ -265,6 +309,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.28.0,<1.0.0" }, { name = "json-repair", specifier = ">=0.39.0,<1.0.0" }, { name = "litellm", specifier = ">=1.79.0,<2.0.0" }, + { name = "minio", specifier = ">=7.2.0,<8.0.0" }, { name = "openai", specifier = ">=1.79.0,<2.0.0" }, { name = "pydantic", specifier = ">=2.12.0,<3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.0,<10.0.0" }, @@ -1420,6 +1465,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "minio" +version = "7.2.20" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/df/6dfc6540f96a74125a11653cce717603fd5b7d0001a8e847b3e54e72d238/minio-7.2.20.tar.gz", hash = "sha256:95898b7a023fbbfde375985aa77e2cd6a0762268db79cf886f002a9ea8e68598", size = 136113, upload-time = "2025-11-27T00:37:15.569Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" }, +] + [[package]] name = "more-itertools" version = "11.0.2" @@ -1759,6 +1820,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" }, + { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" }, + { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" }, + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, +] + [[package]] name = "pydantic" version = "2.13.3" diff --git a/app-instance/create-instance.sh b/app-instance/create-instance.sh index d2a40f1..2b90f18 100755 --- a/app-instance/create-instance.sh +++ b/app-instance/create-instance.sh @@ -15,8 +15,10 @@ CONTAINER_NAME="" HOST_PORT="" PUBLIC_URL="" AUTHZ_BASE_URL="" +AUTHZ_INTERNAL_TOKEN="" AUTHZ_OUTLOOK_MCP_URL="" OUTLOOK_MCP_SERVER_ID="${OUTLOOK_MCP_SERVER_ID:-outlook_mcp}" +USER_FILES_MAX_UPLOAD_BYTES="${USER_FILES_MAX_UPLOAD_BYTES:-}" BACKEND_ID="" CLIENT_ID="" CLIENT_SECRET="" @@ -61,10 +63,14 @@ Optional: --model Model name. Default: openai/gpt-5 --skip-provider-config Create the instance without model/provider/API key settings. --authz-base-url AuthZ service base URL. + --authz-internal-token + AuthZ internal token for backend-only user file storage settings lookup. --authz-outlook-mcp-url Managed Outlook MCP URL for AuthZ mode. --outlook-mcp-server-id Default Outlook MCP server id. Default: outlook_mcp + --user-files-max-upload-bytes + Optional max upload size for the user file system. --backend-id Pre-assigned backend id. --client-id Pre-assigned AuthZ client id. --client-secret Pre-assigned AuthZ client secret. @@ -138,6 +144,7 @@ render_config_json() { API_BASE="$API_BASE" \ SKIP_PROVIDER_CONFIG="$SKIP_PROVIDER_CONFIG" \ AUTHZ_BASE_URL="$AUTHZ_BASE_URL" \ + AUTHZ_INTERNAL_TOKEN="$AUTHZ_INTERNAL_TOKEN" \ AUTHZ_OUTLOOK_MCP_URL="$AUTHZ_OUTLOOK_MCP_URL" \ OUTLOOK_MCP_SERVER_ID="$OUTLOOK_MCP_SERVER_ID" \ BACKEND_ID="$BACKEND_ID" \ @@ -260,6 +267,7 @@ render_runtime_env_file() { TARGET_PATH="$target_path" \ AUTHZ_BASE_URL="$AUTHZ_BASE_URL" \ + AUTHZ_INTERNAL_TOKEN="$AUTHZ_INTERNAL_TOKEN" \ AUTHZ_OUTLOOK_MCP_URL="$AUTHZ_OUTLOOK_MCP_URL" \ BACKEND_ID="$BACKEND_ID" \ CLIENT_ID="$CLIENT_ID" \ @@ -275,6 +283,7 @@ target = Path(os.environ["TARGET_PATH"]) values = { "BEAVER_AUTHZ__ENABLED": "1" if os.environ["AUTHZ_BASE_URL"].strip() else "0", "BEAVER_AUTHZ__BASE_URL": os.environ["AUTHZ_BASE_URL"].strip(), + "BEAVER_AUTHZ_INTERNAL_TOKEN": os.environ["AUTHZ_INTERNAL_TOKEN"].strip(), "BEAVER_AUTHZ__OUTLOOK_MCP_URL": os.environ["AUTHZ_OUTLOOK_MCP_URL"].strip(), "BEAVER_BACKEND_IDENTITY__BACKEND_ID": os.environ["BACKEND_ID"].strip(), "BEAVER_BACKEND_IDENTITY__CLIENT_ID": os.environ["CLIENT_ID"].strip(), @@ -285,6 +294,7 @@ values = { ordered_keys = [ "BEAVER_AUTHZ__ENABLED", "BEAVER_AUTHZ__BASE_URL", + "BEAVER_AUTHZ_INTERNAL_TOKEN", "BEAVER_AUTHZ__OUTLOOK_MCP_URL", "BEAVER_BACKEND_IDENTITY__BACKEND_ID", "BEAVER_BACKEND_IDENTITY__CLIENT_ID", @@ -380,6 +390,10 @@ while [[ $# -gt 0 ]]; do AUTHZ_BASE_URL="${2:-}" shift 2 ;; + --authz-internal-token) + AUTHZ_INTERNAL_TOKEN="${2:-}" + shift 2 + ;; --authz-outlook-mcp-url) AUTHZ_OUTLOOK_MCP_URL="${2:-}" shift 2 @@ -388,6 +402,10 @@ while [[ $# -gt 0 ]]; do OUTLOOK_MCP_SERVER_ID="${2:-}" shift 2 ;; + --user-files-max-upload-bytes) + USER_FILES_MAX_UPLOAD_BYTES="${2:-}" + shift 2 + ;; --backend-id) BACKEND_ID="${2:-}" shift 2 @@ -570,6 +588,10 @@ RUN_ARGS=( --label "beaver.instance.public_url=${PUBLIC_URL}" ) +if [[ -n "$USER_FILES_MAX_UPLOAD_BYTES" ]]; then + RUN_ARGS+=(-e "BEAVER_USER_FILES_MAX_UPLOAD_BYTES=${USER_FILES_MAX_UPLOAD_BYTES}") +fi + if [[ -n "$NETWORK_NAME" ]]; then RUN_ARGS+=(--network "$NETWORK_NAME") fi diff --git a/app-instance/entrypoint.sh b/app-instance/entrypoint.sh index 42f6f84..ad5e876 100755 --- a/app-instance/entrypoint.sh +++ b/app-instance/entrypoint.sh @@ -4,6 +4,9 @@ set -euo pipefail APP_PUBLIC_PORT="${APP_PUBLIC_PORT:-8080}" APP_FRONTEND_PORT="${APP_FRONTEND_PORT:-3000}" APP_BACKEND_PORT="${APP_BACKEND_PORT:-18080}" +UVICORN_LOOP="${UVICORN_LOOP:-asyncio}" +UVICORN_HTTP="${UVICORN_HTTP:-h11}" +UVICORN_WS="${UVICORN_WS:-websockets}" BEAVER_HOME="${BEAVER_HOME:-/root/.beaver}" BEAVER_CONFIG_PATH="${BEAVER_CONFIG_PATH:-$BEAVER_HOME/config.json}" BEAVER_WORKSPACE="${BEAVER_WORKSPACE:-$BEAVER_HOME/workspace}" @@ -59,11 +62,12 @@ export BEAVER_CONFIG_PATH export BEAVER_WORKSPACE export PORT="$APP_FRONTEND_PORT" export HOSTNAME="127.0.0.1" +export PYTHONFAULTHANDLER="${PYTHONFAULTHANDLER:-1}" -log "starting Beaver backend on 127.0.0.1:${APP_BACKEND_PORT}" +log "starting Beaver backend on 127.0.0.1:${APP_BACKEND_PORT} (loop=${UVICORN_LOOP}, http=${UVICORN_HTTP}, ws=${UVICORN_WS})" ( cd /opt/app/backend - python -m uvicorn "beaver.interfaces.web.app:create_app" --factory --host 127.0.0.1 --port "$APP_BACKEND_PORT" + python -m uvicorn "beaver.interfaces.web.app:create_app" --factory --host 127.0.0.1 --port "$APP_BACKEND_PORT" --loop "$UVICORN_LOOP" --http "$UVICORN_HTTP" --ws "$UVICORN_WS" ) & BACKEND_PID=$! diff --git a/app-instance/frontend/app/(app)/files/page.tsx b/app-instance/frontend/app/(app)/files/page.tsx index 984e2a1..70e91d3 100644 --- a/app-instance/frontend/app/(app)/files/page.tsx +++ b/app-instance/frontend/app/(app)/files/page.tsx @@ -21,49 +21,71 @@ import { import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import { - browseWorkspace, - getWorkspaceFile, - getWorkspaceDownloadUrl, - uploadToWorkspace, - deleteWorkspacePath, - createWorkspaceDir, + browseUserFiles, + getUserFile, + getUserFileDownloadUrl, + uploadUserFile, + deleteUserFile, + createUserFileDir, getAccessToken, } from '@/lib/api'; -import type { WorkspaceFileContent, WorkspaceItem } from '@/lib/api'; +import type { UserFileContent, UserFileItem } from '@/lib/api'; import { Button } from '@/components/ui/button'; import { ScrollArea } from '@/components/ui/scroll-area'; import { type AppLocale, pickAppText } from '@/lib/i18n/core'; import { useAppI18n } from '@/lib/i18n/provider'; +const LOAD_RETRY_DELAYS_MS = [0, 600, 1200]; + +function sleep(ms: number): Promise { + return new Promise((resolve) => { + window.setTimeout(resolve, ms); + }); +} + export default function FilesPage() { const { locale } = useAppI18n(); - const [items, setItems] = useState([]); + const [items, setItems] = useState([]); const [currentPath, setCurrentPath] = useState(''); const [loading, setLoading] = useState(true); + const [loadError, setLoadError] = useState(null); const [uploading, setUploading] = useState(false); const [uploadProgress, setUploadProgress] = useState(0); const [showMkdir, setShowMkdir] = useState(false); const [newDirName, setNewDirName] = useState(''); - const [selectedFile, setSelectedFile] = useState(null); + const [selectedFile, setSelectedFile] = useState(null); const [previewLoading, setPreviewLoading] = useState(false); const [previewError, setPreviewError] = useState(null); const fileInputRef = useRef(null); const mkdirInputRef = useRef(null); const load = useCallback(async (path: string = currentPath) => { + let lastError: unknown = null; try { setLoading(true); - const data = await browseWorkspace(path); - setItems(data.items); - setCurrentPath(data.path); - setSelectedFile(null); - setPreviewError(null); - } catch { - // ignore + setLoadError(null); + for (const delay of LOAD_RETRY_DELAYS_MS) { + if (delay > 0) { + await sleep(delay); + } + try { + const data = await browseUserFiles(path); + setItems(data.items); + setCurrentPath(data.path); + setSelectedFile(null); + setPreviewError(null); + return; + } catch (err) { + lastError = err; + } + } + const message = lastError instanceof Error ? lastError.message : pickAppText(locale, '加载文件失败', 'Failed to load files'); + setLoadError(message); + setItems([]); } finally { setLoading(false); } - }, [currentPath]); + }, [currentPath, locale]); useEffect(() => { load(''); @@ -73,12 +95,12 @@ export default function FilesPage() { load(path); }; - const openFile = async (item: WorkspaceItem) => { + const openFile = async (item: UserFileItem) => { if (item.type !== 'file') return; setPreviewLoading(true); setPreviewError(null); try { - setSelectedFile(await getWorkspaceFile(item.path)); + setSelectedFile(await getUserFile(item.path)); } catch (err: any) { setPreviewError(err.message || pickAppText(locale, '加载文件失败', 'Failed to load file')); setSelectedFile(null); @@ -87,7 +109,7 @@ export default function FilesPage() { } }; - const handleDelete = async (item: WorkspaceItem) => { + const handleDelete = async (item: UserFileItem) => { const label = item.type === 'directory' ? pickAppText(locale, '文件夹', 'folder') : pickAppText(locale, '文件', 'file'); @@ -99,7 +121,7 @@ export default function FilesPage() { return; } try { - await deleteWorkspacePath(item.path); + await deleteUserFile(item.path); setItems((prev) => prev.filter((i) => i.path !== item.path)); if (selectedFile?.path === item.path) { setSelectedFile(null); @@ -109,8 +131,8 @@ export default function FilesPage() { } }; - const handleDownload = async (item: WorkspaceItem) => { - const url = getWorkspaceDownloadUrl(item.path); + const handleDownload = async (item: UserFileItem) => { + const url = getUserFileDownloadUrl(item.path); const token = getAccessToken(); const headers: Record = {}; if (token) headers['Authorization'] = `Bearer ${token}`; @@ -138,7 +160,7 @@ export default function FilesPage() { setUploadProgress(0); try { for (let i = 0; i < files.length; i++) { - await uploadToWorkspace(files[i], currentPath, (pct) => { + await uploadUserFile(files[i], currentPath || 'uploads', (pct) => { setUploadProgress(Math.round((i / files.length) * 100 + pct / files.length)); }); } @@ -157,7 +179,7 @@ export default function FilesPage() { if (!name) return; try { const dirPath = currentPath ? `${currentPath}/${name}` : name; - await createWorkspaceDir(dirPath); + await createUserFileDir(dirPath); setShowMkdir(false); setNewDirName(''); await load(); @@ -176,7 +198,8 @@ export default function FilesPage() { return `${bytes} B`; }; - const formatDate = (iso: string) => { + const formatDate = (iso: string | null | undefined) => { + if (!iso) return ''; try { return new Date(iso).toLocaleString(locale, { month: '2-digit', @@ -199,7 +222,7 @@ export default function FilesPage() { variant="outline" size="sm" onClick={() => setShowMkdir(true)} - disabled={loading} + disabled={loading || !currentPath} > {pickAppText(locale, '新建文件夹', 'New folder')} @@ -208,7 +231,7 @@ export default function FilesPage() { variant="outline" size="sm" onClick={() => fileInputRef.current?.click()} - disabled={uploading} + disabled={uploading || !currentPath} > {uploading ? ( <> @@ -246,7 +269,7 @@ export default function FilesPage() { className="flex items-center gap-1 hover:text-foreground transition-colors px-1.5 py-0.5 rounded hover:bg-accent" > - {pickAppText(locale, '工作区', 'Workspace')} + {pickAppText(locale, '文件', 'Files')} {breadcrumbs.map((segment, idx) => { const path = breadcrumbs.slice(0, idx + 1).join('/'); @@ -312,6 +335,16 @@ export default function FilesPage() {
+ ) : loadError ? ( +
+ +

{pickAppText(locale, '加载失败', 'Failed to load')}

+

{loadError}

+ +
) : items.length === 0 ? (
@@ -340,7 +373,7 @@ export default function FilesPage() { {item.type === 'directory' ? ( ) : ( - + )}
@@ -412,7 +445,7 @@ export default function FilesPage() { error={previewError} formatSize={formatSize} formatDate={formatDate} - downloadUrl={selectedFile ? getWorkspaceDownloadUrl(selectedFile.path) : null} + downloadUrl={selectedFile ? getUserFileDownloadUrl(selectedFile.path) : null} locale={locale} /> @@ -429,11 +462,11 @@ function FilePreviewPanel({ downloadUrl, locale, }: { - file: WorkspaceFileContent | null; + file: UserFileContent | null; loading: boolean; error: string | null; formatSize: (bytes: number | null) => string; - formatDate: (iso: string) => string; + formatDate: (iso: string | null | undefined) => string; downloadUrl: string | null; locale: AppLocale; }) { @@ -516,10 +549,10 @@ function FileIcon({ name, contentType }: { name: string; contentType?: string }) return ; } -function isImage(file: WorkspaceFileContent): boolean { +function isImage(file: UserFileContent): boolean { return file.content_type.startsWith('image/'); } -function isMarkdown(file: WorkspaceFileContent): boolean { +function isMarkdown(file: UserFileContent): boolean { return file.path.toLowerCase().endsWith('.md') || file.content_type.includes('markdown'); } diff --git a/app-instance/frontend/app/(app)/mcp/page.tsx b/app-instance/frontend/app/(app)/mcp/page.tsx index 08ac84e..16da65b 100644 --- a/app-instance/frontend/app/(app)/mcp/page.tsx +++ b/app-instance/frontend/app/(app)/mcp/page.tsx @@ -97,6 +97,16 @@ function transportLabel(transport: string | undefined, locale: AppLocale) { return transport || '-'; } +function discoveredToolCount( + serverId: string, + tools: Array<{ server_id: string; tools: Array> }>, + fallback?: number, +) { + const group = tools.find((item) => item.server_id === serverId); + if (group) return group.tools.length; + return fallback || 0; +} + export default function MCPPage() { const { locale } = useAppI18n(); const t = (zh: string, en: string) => pickAppText(locale, zh, en); @@ -543,7 +553,7 @@ export default function MCPPage() {
Scopes： {t('由 AuthZ 动态决定', 'Derived from AuthZ')}
)}
- {t(`${server.tool_count || 0} 个工具`, `${server.tool_count || 0} tools`)} + {t(`${discoveredToolCount(server.id, tools, server.tool_count)} 个工具`, `${discoveredToolCount(server.id, tools, server.tool_count)} tools`)} {selectedServerId === server.id ? t('已选中', 'Selected') : t('点击查看工具', 'Click to view tools')} {server.last_error && {server.last_error}}
diff --git a/app-instance/frontend/components/AppRuntimeBridge.tsx b/app-instance/frontend/components/AppRuntimeBridge.tsx index 714ae1c..ac97476 100644 --- a/app-instance/frontend/components/AppRuntimeBridge.tsx +++ b/app-instance/frontend/components/AppRuntimeBridge.tsx @@ -1,6 +1,7 @@ 'use client'; import React from 'react'; +import { usePathname } from 'next/navigation'; import { getStatus, listSessions, wsManager } from '@/lib/api'; import { useChatStore } from '@/lib/store'; @@ -37,6 +38,7 @@ function isSessionUpdatedEvent(data: WsEvent | Record): data is } export function AppRuntimeBridge() { + const pathname = usePathname(); const sessionId = useChatStore((state) => state.sessionId); const setSessions = useChatStore((state) => state.setSessions); const setWsStatus = useChatStore((state) => state.setWsStatus); @@ -45,6 +47,7 @@ export function AppRuntimeBridge() { const ingestProcessEvent = useChatStore((state) => state.ingestProcessEvent); const statusCheckCleanupRef = React.useRef<(() => void) | null>(null); const statusCheckInFlightRef = React.useRef(false); + const chatRuntimeEnabled = pathname === '/' || pathname.startsWith('/tasks') || pathname.startsWith('/notifications'); const loadSessions = React.useCallback(async () => { try { @@ -73,15 +76,27 @@ export function AppRuntimeBridge() { }, [setBeaverReady]); React.useEffect(() => { + if (!chatRuntimeEnabled) { + return; + } void loadSessions(); - }, [loadSessions]); + }, [chatRuntimeEnabled, loadSessions]); React.useEffect(() => { + if (!chatRuntimeEnabled) { + wsManager.disconnect(); + setWsStatus('disconnected'); + setBeaverReady(null); + return; + } resetProcessState(); wsManager.connect(sessionId); - }, [resetProcessState, sessionId]); + }, [chatRuntimeEnabled, resetProcessState, sessionId, setBeaverReady, setWsStatus]); React.useEffect(() => { + if (!chatRuntimeEnabled) { + return; + } const unsubStatus = wsManager.onStatusChange((status) => { setWsStatus(status); if (status === 'connected') { @@ -98,9 +113,12 @@ export function AppRuntimeBridge() { statusCheckCleanupRef.current = null; unsubStatus(); }; - }, [scheduleStatusCheck, setBeaverReady, setWsStatus]); + }, [chatRuntimeEnabled, scheduleStatusCheck, setBeaverReady, setWsStatus]); React.useEffect(() => { + if (!chatRuntimeEnabled) { + return; + } const unsubMessage = wsManager.onMessage((data) => { if (isSessionUpdatedEvent(data)) { void loadSessions(); @@ -115,7 +133,7 @@ export function AppRuntimeBridge() { return () => { unsubMessage(); }; - }, [ingestProcessEvent, loadSessions]); + }, [chatRuntimeEnabled, ingestProcessEvent, loadSessions]); return null; } diff --git a/app-instance/frontend/lib/api.ts b/app-instance/frontend/lib/api.ts index b24aaa3..0649320 100644 --- a/app-instance/frontend/lib/api.ts +++ b/app-instance/frontend/lib/api.ts @@ -1363,3 +1363,112 @@ export async function createWorkspaceDir(path: string): Promise { method: 'POST', }); } + +// --------------------------------------------------------------------------- +// User File System +// --------------------------------------------------------------------------- + +export interface UserFileItem { + name: string; + path: string; + type: 'file' | 'directory'; + size: number | null; + content_type?: string | null; + modified?: string | null; +} + +export interface UserFileBrowseResult { + path: string; + items: UserFileItem[]; +} + +export interface UserFileContent { + name: string; + path: string; + size: number; + content_type: string; + modified: string | null; + is_binary: boolean; + is_truncated: boolean; + content: string | null; +} + +export interface UserFilesStatus { + configured: boolean; + storage_mode: string; + roots: string[]; + workspace_visible: boolean; +} + +export async function getUserFilesStatus(): Promise { + return fetchJSON('/api/user-files/status'); +} + +export async function browseUserFiles(path: string = ''): Promise { + const params = path ? `?path=${encodeURIComponent(path)}` : ''; + return fetchJSON(`/api/user-files/browse${params}`); +} + +export async function getUserFile(path: string): Promise { + return fetchJSON(`/api/user-files/preview?path=${encodeURIComponent(path)}`); +} + +export function getUserFileDownloadUrl(path: string): string { + return buildApiUrl(`/api/user-files/download?path=${encodeURIComponent(path)}`); +} + +export async function uploadUserFile( + file: File, + dirPath: string = 'uploads', + onProgress?: (percent: number) => void +): Promise { + const locale = getCurrentAppLocale(); + const formData = new FormData(); + formData.append('file', file); + formData.append('path', dirPath); + + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open('POST', buildApiUrl('/api/user-files/upload')); + const token = getAccessToken(); + if (token) { + xhr.setRequestHeader('Authorization', `Bearer ${token}`); + } + + xhr.upload.onprogress = (e) => { + if (e.lengthComputable && onProgress) { + onProgress(Math.round((e.loaded / e.total) * 100)); + } + }; + + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + resolve(JSON.parse(xhr.responseText)); + } else { + let detail = ''; + try { + const data = JSON.parse(xhr.responseText); + detail = typeof data?.detail === 'string' ? data.detail : ''; + } catch { + detail = ''; + } + reject(new Error(detail || `${pickAppText(locale, '上传失败', 'Upload failed')}: ${xhr.status}`)); + } + }; + + xhr.onerror = () => reject(new Error(pickAppText(locale, '上传失败', 'Upload failed'))); + xhr.send(formData); + }); +} + +export async function deleteUserFile(path: string): Promise { + await fetchJSON(`/api/user-files/delete?path=${encodeURIComponent(path)}`, { + method: 'DELETE', + }); +} + +export async function createUserFileDir(path: string): Promise { + return fetchJSON(`/api/user-files/mkdir?path=${encodeURIComponent(path)}`, { + method: 'POST', + }); +} diff --git a/app-instance/frontend/lib/user-files-api.test.ts b/app-instance/frontend/lib/user-files-api.test.ts new file mode 100644 index 0000000..4dc3c04 --- /dev/null +++ b/app-instance/frontend/lib/user-files-api.test.ts @@ -0,0 +1,32 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +const root = resolve(__dirname, '..'); + +describe('user file system frontend wiring', () => { + it('routes API client helpers to user file endpoints', () => { + const apiSource = readFileSync(resolve(root, 'lib/api.ts'), 'utf8'); + + expect(apiSource).toContain('/api/user-files/browse'); + expect(apiSource).toContain('/api/user-files/upload'); + expect(apiSource).toContain('/api/user-files/download'); + expect(apiSource).toContain('/api/user-files/preview'); + expect(apiSource).toContain('/api/user-files/delete'); + expect(apiSource).toContain('/api/user-files/mkdir'); + }); + + it('does not wire the Files page to workspace or MinIO management APIs', () => { + const pageSource = readFileSync(resolve(root, 'app/(app)/files/page.tsx'), 'utf8'); + + expect(pageSource).toContain('browseUserFiles'); + expect(pageSource).toContain('uploadUserFile'); + expect(pageSource).not.toContain('browseWorkspace'); + expect(pageSource).not.toContain('uploadToWorkspace'); + expect(pageSource).not.toContain('MinIO'); + expect(pageSource).not.toContain('bucket'); + expect(pageSource).not.toContain('accessKey'); + expect(pageSource).not.toContain('secretKey'); + }); +}); diff --git a/app-instance/instance-registry.py b/app-instance/instance-registry.py index e83edcb..b8cc205 100755 --- a/app-instance/instance-registry.py +++ b/app-instance/instance-registry.py @@ -46,6 +46,21 @@ def _normalize_record(record: dict[str, Any]) -> dict[str, Any]: return normalized +def read_registry(path: Path) -> dict[str, Any]: + if not path.exists(): + return _default_data() + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return _default_data() + if not isinstance(data, dict): + return _default_data() + if not isinstance(data.get("instances"), list): + data["instances"] = [] + data["instances"] = [_normalize_record(item) for item in data["instances"] if isinstance(item, dict)] + return data + + @contextmanager def locked_registry(path: Path): path.parent.mkdir(parents=True, exist_ok=True) @@ -118,8 +133,8 @@ def _get_record( def cmd_list(args: argparse.Namespace) -> int: path = Path(args.registry).expanduser() - with locked_registry(path) as data: - instances = list(data["instances"]) + data = read_registry(path) + instances = list(data["instances"]) if args.json: json.dump({"instances": instances}, sys.stdout, indent=2, ensure_ascii=False) sys.stdout.write("\n") @@ -143,15 +158,15 @@ def cmd_list(args: argparse.Namespace) -> int: def cmd_get(args: argparse.Namespace) -> int: path = Path(args.registry).expanduser() - with locked_registry(path) as data: - record = _get_record( - data, - instance_id=args.instance_id, - slug=args.slug, - container_name=args.container_name, - username=args.username, - instance_host=args.instance_host, - ) + data = read_registry(path) + record = _get_record( + data, + instance_id=args.instance_id, + slug=args.slug, + container_name=args.container_name, + username=args.username, + instance_host=args.instance_host, + ) if record is None: return 1 json.dump(record, sys.stdout, indent=2, ensure_ascii=False) diff --git a/app-instance/nginx.conf b/app-instance/nginx.conf index 105b22e..d46d85e 100644 --- a/app-instance/nginx.conf +++ b/app-instance/nginx.conf @@ -11,7 +11,7 @@ http { sendfile on; tcp_nopush on; keepalive_timeout 65; - client_max_body_size 50m; + client_max_body_size 5g; access_log /dev/stdout; error_log /dev/stderr warn; @@ -69,4 +69,3 @@ http { } } } - diff --git a/authz-service/src/app/json_store.py b/authz-service/src/app/json_store.py index 593d76c..be5f7b1 100644 --- a/authz-service/src/app/json_store.py +++ b/authz-service/src/app/json_store.py @@ -7,7 +7,15 @@ from pathlib import Path from threading import Lock from typing import Any -from app.models import ChannelSettings, BackendCredential, BackendRecord, OutlookSettings, UserRecord, utcnow_iso +from app.models import ( + ChannelSettings, + BackendCredential, + BackendRecord, + MinIOSettings, + OutlookSettings, + UserRecord, + utcnow_iso, +) class JsonStore: @@ -152,6 +160,19 @@ class JsonStore: return None return OutlookSettings.model_validate(outlook) + def get_minio_settings(self, backend_id: str) -> MinIOSettings | None: + raw = self._read_json(self.settings_path, {"settings": {}}) + root = raw.get("settings", {}) + if not isinstance(root, dict): + return None + backend_settings = root.get(backend_id, {}) + if not isinstance(backend_settings, dict): + return None + minio = backend_settings.get("minio") + if not isinstance(minio, dict): + return None + return MinIOSettings.model_validate(minio) + def list_channel_settings(self, backend_id: str) -> dict[str, ChannelSettings]: raw = self._read_json(self.settings_path, {"settings": {}}) root = raw.get("settings", {}) @@ -185,6 +206,19 @@ class JsonStore: self._write_json(self.settings_path, {"settings": root}) return backend_settings + def save_minio_settings(self, backend_id: str, settings: MinIOSettings) -> dict[str, Any]: + raw = self._read_json(self.settings_path, {"settings": {}}) + root = raw.get("settings", {}) + if not isinstance(root, dict): + root = {} + backend_settings = root.get(backend_id, {}) + if not isinstance(backend_settings, dict): + backend_settings = {} + backend_settings["minio"] = settings.model_dump(mode="json") + root[backend_id] = backend_settings + self._write_json(self.settings_path, {"settings": root}) + return backend_settings + def save_channel_settings(self, backend_id: str, channel_id: str, settings: ChannelSettings) -> dict[str, Any]: raw = self._read_json(self.settings_path, {"settings": {}}) root = raw.get("settings", {}) @@ -218,6 +252,22 @@ class JsonStore: self._write_json(self.settings_path, {"settings": root}) return True + def delete_minio_settings(self, backend_id: str) -> bool: + raw = self._read_json(self.settings_path, {"settings": {}}) + root = raw.get("settings", {}) + if not isinstance(root, dict): + return False + backend_settings = root.get(backend_id) + if not isinstance(backend_settings, dict) or "minio" not in backend_settings: + return False + backend_settings.pop("minio", None) + if backend_settings: + root[backend_id] = backend_settings + else: + root.pop(backend_id, None) + self._write_json(self.settings_path, {"settings": root}) + return True + def delete_channel_settings(self, backend_id: str, channel_id: str) -> bool: raw = self._read_json(self.settings_path, {"settings": {}}) root = raw.get("settings", {}) diff --git a/authz-service/src/app/main.py b/authz-service/src/app/main.py index ee8a145..ec91bc2 100644 --- a/authz-service/src/app/main.py +++ b/authz-service/src/app/main.py @@ -1,12 +1,13 @@ from __future__ import annotations +import asyncio import os import re from pathlib import Path from typing import Any import httpx -from fastapi import Depends, FastAPI, Header, HTTPException, Request +from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request from fastapi.responses import JSONResponse from app.json_store import JsonStore @@ -16,6 +17,7 @@ from app.models import ( BackendRecord, IntrospectRequest, IntrospectResponse, + MinIOSettings, OAuthTokenRequest, OAuthTokenResponse, OutlookSettings, @@ -30,6 +32,12 @@ from app.models import ( UserRecord, utcnow_iso, ) +from app.minio_provisioning import ( + MinIODeprovisioningError, + MinIOProvisioningError, + deprovision_user_file_minio_resources, + provision_user_file_minio_settings, +) from app.security import JwtSigner, generate_client_secret, hash_secret, verify_secret DATA_DIR = Path(os.getenv("AUTHZ_DATA_DIR", Path(__file__).resolve().parents[1] / "data")) @@ -360,6 +368,42 @@ def _upsert_user( return user +async def _ensure_user_file_storage_settings(backend_id: str) -> MinIOSettings | None: + try: + settings = await asyncio.to_thread( + provision_user_file_minio_settings, + backend_id=backend_id, + existing=store.get_minio_settings(backend_id), + ) + except MinIOProvisioningError as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + if settings is not None: + store.save_minio_settings(backend_id, settings) + store.touch_backend(backend_id) + return settings + + +async def _deprovision_user_file_storage(backend_id: str, *, best_effort: bool = False) -> dict[str, Any]: + existing = store.get_minio_settings(backend_id) + try: + result = await asyncio.to_thread( + deprovision_user_file_minio_resources, + backend_id=backend_id, + existing=existing, + best_effort=best_effort, + ) + except MinIODeprovisioningError as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + + settings_removed = False + if result.get("ok") is True: + settings_removed = store.delete_minio_settings(backend_id) + if settings_removed: + store.touch_backend(backend_id) + result["settings"] = {"status": "removed" if settings_removed else "absent"} + return result + + def _resolve_register_backend_payload(req: RegisterBackendRequest) -> tuple[str, str, str, str | None]: backend_name = req.name.strip() or "backend" backend_id = _clean_optional(req.backend_id) @@ -475,6 +519,7 @@ async def register_backend(req: RegisterBackendRequest) -> RegisterBackendRespon base_url=base_url, frontend_base_url=frontend_base_url, ) + await _ensure_user_file_storage_settings(backend.backend_id) return RegisterBackendResponse( backend_id=backend.backend_id, client_id=backend.backend_id, @@ -589,6 +634,39 @@ async def delete_outlook_settings(backend_id: str) -> dict[str, Any]: return {"ok": removed} +@app.get("/backends/{backend_id}/settings/minio") +async def get_minio_settings(backend_id: str) -> dict[str, Any]: + _ensure_backend(backend_id) + settings = store.get_minio_settings(backend_id) + if settings is None: + return {"configured": False} + return settings.masked_dict() + + +@app.post("/backends/{backend_id}/settings/minio") +async def save_minio_settings(backend_id: str, payload: MinIOSettings) -> dict[str, Any]: + _ensure_backend(backend_id) + existing = store.get_minio_settings(backend_id) + if not payload.secret_key: + if existing is None or not existing.secret_key: + raise HTTPException(status_code=400, detail="Secret key is required for initial MinIO setup") + payload = payload.model_copy(update={"secret_key": existing.secret_key, "updated_at": utcnow_iso()}) + else: + payload = payload.model_copy(update={"updated_at": utcnow_iso()}) + store.save_minio_settings(backend_id, payload) + store.touch_backend(backend_id) + return payload.masked_dict() + + +@app.delete("/backends/{backend_id}/settings/minio") +async def delete_minio_settings(backend_id: str) -> dict[str, Any]: + _ensure_backend(backend_id) + removed = store.delete_minio_settings(backend_id) + if removed: + store.touch_backend(backend_id) + return {"ok": removed} + + @app.get("/backends/{backend_id}/settings/channels") async def list_channel_settings(backend_id: str) -> dict[str, Any]: _ensure_backend(backend_id) @@ -662,6 +740,7 @@ async def oauth_register(req: RegisterUserRequest) -> RegisterUserResponse: email=_clean_optional(req.email), default_backend_id=backend.backend_id, ) + await _ensure_user_file_storage_settings(backend.backend_id) return RegisterUserResponse( user=user, backend=RegisterUserBackendResult( @@ -686,6 +765,23 @@ async def get_internal_outlook_settings(backend_id: str) -> dict[str, Any]: return settings.model_dump(mode="json") +@app.get("/internal/backends/{backend_id}/settings/minio", dependencies=[Depends(_require_internal)]) +async def get_internal_minio_settings(backend_id: str) -> dict[str, Any]: + _ensure_backend(backend_id) + settings = store.get_minio_settings(backend_id) + if settings is None: + raise HTTPException(status_code=404, detail="MinIO settings not configured") + return settings.model_dump(mode="json") + + +@app.delete("/internal/backends/{backend_id}/user-files", dependencies=[Depends(_require_internal)]) +async def delete_internal_user_files( + backend_id: str, + best_effort: bool = Query(default=False), +) -> dict[str, Any]: + return await _deprovision_user_file_storage(backend_id, best_effort=best_effort) + + @app.get("/internal/backends/{backend_id}/settings/channels", dependencies=[Depends(_require_internal)]) async def list_internal_channel_settings(backend_id: str) -> dict[str, Any]: _ensure_backend(backend_id) diff --git a/authz-service/src/app/minio_provisioning.py b/authz-service/src/app/minio_provisioning.py new file mode 100644 index 0000000..74e1137 --- /dev/null +++ b/authz-service/src/app/minio_provisioning.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +import re +import secrets +from typing import Any + +from .models import MinIOSettings + + +@dataclass(slots=True) +class MinIOProvisioningConfig: + enabled: bool + endpoint: str + public_endpoint: str + admin_access_key: str + admin_secret_key: str + bucket: str + secure: bool + region: str | None + + +class MinIOProvisioningError(RuntimeError): + pass + + +class MinIODeprovisioningError(RuntimeError): + pass + + +def minio_provisioning_config_from_env() -> MinIOProvisioningConfig: + endpoint = _env("USER_FILES_MINIO_ENDPOINT") or _env("MINIO_ENDPOINT") + public_endpoint = _env("USER_FILES_MINIO_PUBLIC_ENDPOINT") or endpoint + return MinIOProvisioningConfig( + enabled=_truthy(_env("USER_FILES_MINIO_PROVISIONING_ENABLED")), + endpoint=endpoint, + public_endpoint=public_endpoint, + admin_access_key=_env("USER_FILES_MINIO_ADMIN_ACCESS_KEY") or _env("MINIO_ROOT_USER"), + admin_secret_key=_env("USER_FILES_MINIO_ADMIN_SECRET_KEY") or _env("MINIO_ROOT_PASSWORD"), + bucket=_env("USER_FILES_MINIO_BUCKET") or "beaver-user-files", + secure=_truthy(_env("USER_FILES_MINIO_SECURE")), + region=_env("USER_FILES_MINIO_REGION") or None, + ) + + +def provision_user_file_minio_settings( + *, + backend_id: str, + existing: MinIOSettings | None, + config: MinIOProvisioningConfig | None = None, +) -> MinIOSettings | None: + cfg = config or minio_provisioning_config_from_env() + if not cfg.enabled: + return existing + if existing is not None and existing.configured and existing.secret_key: + return existing + if not cfg.endpoint or not cfg.admin_access_key or not cfg.admin_secret_key: + raise MinIOProvisioningError("MinIO provisioning requires endpoint and admin credentials") + + namespace = default_namespace(backend_id) + access_key = _access_key_for_backend(backend_id) + secret_key = secrets.token_urlsafe(32) + policy_name = _policy_name_for_backend(backend_id) + + try: + from minio import Minio + from minio import MinioAdmin + from minio.credentials import StaticProvider + + client = Minio( + endpoint=cfg.endpoint, + access_key=cfg.admin_access_key, + secret_key=cfg.admin_secret_key, + secure=cfg.secure, + region=cfg.region, + ) + if not client.bucket_exists(cfg.bucket): + client.make_bucket(cfg.bucket, location=cfg.region) + + admin = MinioAdmin( + endpoint=cfg.endpoint, + credentials=StaticProvider( + access_key=cfg.admin_access_key, + secret_key=cfg.admin_secret_key, + ), + secure=cfg.secure, + ) + try: + admin.user_add(access_key, secret_key) + except Exception: + # Treat an existing user as idempotent; keep the newly generated + # secret only when user creation succeeded. + if existing is not None and existing.secret_key: + secret_key = existing.secret_key + else: + raise + + policy = _namespace_policy(bucket=cfg.bucket, namespace=namespace) + admin.policy_add(policy_name, policy=policy) + admin.attach_policy(policies=[policy_name], user=access_key) + except Exception as exc: + raise MinIOProvisioningError(f"MinIO user file provisioning failed: {exc}") from exc + + return MinIOSettings( + endpoint=cfg.public_endpoint, + access_key=access_key, + secret_key=secret_key, + bucket=cfg.bucket, + namespace=namespace, + secure=cfg.secure, + region=cfg.region, + ) + + +def deprovision_user_file_minio_resources( + *, + backend_id: str, + existing: MinIOSettings | None, + best_effort: bool = False, + config: MinIOProvisioningConfig | None = None, +) -> dict[str, Any]: + cfg = config or minio_provisioning_config_from_env() + settings_found = existing is not None + result: dict[str, Any] = { + "ok": True, + "backend_id": backend_id, + "settings_found": settings_found, + "best_effort": best_effort, + "bucket": "skipped", + "objects": {"status": "absent", "deleted": 0}, + "user": {"status": "absent"}, + "policy": {"status": "absent"}, + } + + if existing is None and not best_effort: + return result + + bucket = (existing.bucket if existing else None) or cfg.bucket + namespace = (existing.namespace if existing else None) or default_namespace(backend_id) + access_key = (existing.access_key if existing else None) or _access_key_for_backend(backend_id) + policy_name = _policy_name_for_backend(backend_id) + + if not bucket or not namespace.strip("/"): + raise MinIODeprovisioningError("MinIO deprovisioning requires bucket and namespace") + if not cfg.endpoint or not cfg.admin_access_key or not cfg.admin_secret_key: + raise MinIODeprovisioningError("MinIO deprovisioning requires endpoint and admin credentials") + + try: + from minio import Minio + from minio import MinioAdmin + from minio.credentials import StaticProvider + from minio.deleteobjects import DeleteObject + + client = Minio( + endpoint=cfg.endpoint, + access_key=cfg.admin_access_key, + secret_key=cfg.admin_secret_key, + secure=cfg.secure, + region=cfg.region, + ) + admin = MinioAdmin( + endpoint=cfg.endpoint, + credentials=StaticProvider( + access_key=cfg.admin_access_key, + secret_key=cfg.admin_secret_key, + ), + secure=cfg.secure, + ) + + prefix = namespace.strip("/") + if client.bucket_exists(bucket): + result["bucket"] = "present" + objects = [ + DeleteObject(item.object_name) + for item in client.list_objects(bucket, prefix=f"{prefix}/", recursive=True) + if getattr(item, "object_name", None) + ] + errors = list(client.remove_objects(bucket, objects)) if objects else [] + if errors: + result["objects"] = { + "status": "failed", + "deleted": 0, + "errors": [_safe_error_text(error) for error in errors], + } + result["ok"] = False + else: + result["objects"] = { + "status": "removed" if objects else "absent", + "deleted": len(objects), + } + else: + result["bucket"] = "absent" + result["objects"] = {"status": "absent", "deleted": 0} + + result["policy_detach"] = _admin_step( + lambda: _call_admin_method( + admin, + ["detach_policy"], + policies=[policy_name], + user=access_key, + ) + ) + result["user"] = _admin_step(lambda: _call_admin_method(admin, ["user_remove", "remove_user"], access_key)) + result["policy"] = _admin_step( + lambda: _call_admin_method(admin, ["policy_remove", "remove_policy"], policy_name) + ) + except Exception as exc: + raise MinIODeprovisioningError(f"MinIO user file deprovisioning failed: {_safe_error_text(exc)}") from exc + + result["ok"] = bool(result["ok"]) and all( + item.get("status") != "failed" + for item in ( + result["objects"], + result["policy_detach"], + result["user"], + result["policy"], + ) + if isinstance(item, dict) + ) + return result + + +def default_namespace(backend_id: str) -> str: + return f"users/{backend_id.strip().strip('/')}" + + +def _namespace_policy(*, bucket: str, namespace: str) -> dict[str, object]: + prefix = namespace.strip("/") + return { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": [f"arn:aws:s3:::{bucket}"], + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": [f"arn:aws:s3:::{bucket}"], + "Condition": {"StringLike": {"s3:prefix": [f"{prefix}/*", prefix]}}, + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + ], + "Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*"], + }, + ], + } + + +def policy_json_for_backend(*, backend_id: str, bucket: str = "beaver-user-files") -> str: + return json.dumps(_namespace_policy(bucket=bucket, namespace=default_namespace(backend_id)), indent=2) + + +def _access_key_for_backend(backend_id: str) -> str: + slug = re.sub(r"[^A-Za-z0-9_-]+", "-", backend_id.strip()).strip("-_") + if not slug: + slug = secrets.token_hex(6) + return f"beaver-{slug}"[:64] + + +def _policy_name_for_backend(backend_id: str) -> str: + slug = re.sub(r"[^A-Za-z0-9_-]+", "-", backend_id.strip()).strip("-_") + return f"beaver-user-files-{slug or secrets.token_hex(6)}"[:128] + + +def _call_admin_method(admin: object, names: list[str], *args: Any, **kwargs: Any) -> Any: + for name in names: + method = getattr(admin, name, None) + if callable(method): + return method(*args, **kwargs) + raise AttributeError(f"MinIO admin client does not support any of: {', '.join(names)}") + + +def _admin_step(action: Any) -> dict[str, Any]: + try: + action() + except Exception as exc: + if _is_absent_error(exc): + return {"status": "absent"} + return {"status": "failed", "error": _safe_error_text(exc)} + return {"status": "removed"} + + +def _is_absent_error(exc: Exception) -> bool: + text = _safe_error_text(exc).lower() + absent_markers = ( + "not found", + "notfound", + "no such", + "does not exist", + "doesn't exist", + "specified user does not exist", + "specified policy does not exist", + "the specified bucket does not exist", + ) + return any(marker in text for marker in absent_markers) + + +def _safe_error_text(exc: object) -> str: + text = str(exc).strip() + return text or exc.__class__.__name__ + + +def _env(name: str) -> str: + return os.getenv(name, "").strip() + + +def _truthy(value: str) -> bool: + return value.lower() in {"1", "true", "yes", "on"} diff --git a/authz-service/src/app/models.py b/authz-service/src/app/models.py index aa30c05..1ab73c5 100644 --- a/authz-service/src/app/models.py +++ b/authz-service/src/app/models.py @@ -77,6 +77,24 @@ class OutlookSettings(BaseModel): return data +class MinIOSettings(BaseModel): + configured: bool = True + endpoint: str + access_key: str + secret_key: str + bucket: str | None = None + namespace: str | None = None + secure: bool = False + region: str | None = None + updated_at: str = Field(default_factory=utcnow_iso) + + def masked_dict(self) -> dict[str, Any]: + data = self.model_dump() + data.pop("secret_key", None) + data["secret_key_masked"] = True + return data + + class BackendRoutingPayload(BaseModel): name: str | None = None backend_id: str | None = None diff --git a/authz-service/src/pyproject.toml b/authz-service/src/pyproject.toml index 0906871..237a7b8 100644 --- a/authz-service/src/pyproject.toml +++ b/authz-service/src/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "httpx>=0.28.0,<1.0.0", "pydantic>=2.12.0,<3.0.0", "cryptography>=45.0.0,<46.0.0", + "minio>=7.2.0,<8.0.0", "PyJWT>=2.10.0,<3.0.0", "python-multipart>=0.0.20,<1.0.0", ] diff --git a/authz-service/src/tests/test_minio_deprovisioning.py b/authz-service/src/tests/test_minio_deprovisioning.py new file mode 100644 index 0000000..da39f03 --- /dev/null +++ b/authz-service/src/tests/test_minio_deprovisioning.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import importlib +import sys +from types import ModuleType +from typing import Any + +from fastapi.testclient import TestClient + +from app.minio_provisioning import ( + MinIOProvisioningConfig, + deprovision_user_file_minio_resources, +) +from app.models import MinIOSettings + + +class _FakeObject: + def __init__(self, object_name: str) -> None: + self.object_name = object_name + + +class _FakeMinio: + bucket_exists_value = True + objects: list[str] = [] + removed_objects: list[str] = [] + + def __init__(self, **_kwargs: Any) -> None: + pass + + def bucket_exists(self, bucket: str) -> bool: + return self.bucket_exists_value + + def list_objects(self, bucket: str, *, prefix: str, recursive: bool) -> list[_FakeObject]: + return [_FakeObject(name) for name in self.objects if name.startswith(prefix)] + + def remove_objects(self, bucket: str, objects: list[Any]) -> list[Any]: + self.removed_objects.extend(item.object_name for item in objects) + return [] + + +class _FakeAdmin: + calls: list[tuple[str, Any]] = [] + missing = False + + def __init__(self, **_kwargs: Any) -> None: + pass + + def detach_policy(self, **kwargs: Any) -> None: + self.calls.append(("detach_policy", kwargs)) + if self.missing: + raise RuntimeError("policy not found") + + def user_remove(self, access_key: str) -> None: + self.calls.append(("user_remove", access_key)) + if self.missing: + raise RuntimeError("user not found") + + def policy_remove(self, policy_name: str) -> None: + self.calls.append(("policy_remove", policy_name)) + if self.missing: + raise RuntimeError("policy not found") + + +class _FakeStaticProvider: + def __init__(self, **_kwargs: Any) -> None: + pass + + +class _FakeDeleteObject: + def __init__(self, object_name: str) -> None: + self.object_name = object_name + + +def _install_fake_minio(monkeypatch) -> None: + minio_module = ModuleType("minio") + minio_module.Minio = _FakeMinio + minio_module.MinioAdmin = _FakeAdmin + + credentials_module = ModuleType("minio.credentials") + credentials_module.StaticProvider = _FakeStaticProvider + + deleteobjects_module = ModuleType("minio.deleteobjects") + deleteobjects_module.DeleteObject = _FakeDeleteObject + + monkeypatch.setitem(sys.modules, "minio", minio_module) + monkeypatch.setitem(sys.modules, "minio.credentials", credentials_module) + monkeypatch.setitem(sys.modules, "minio.deleteobjects", deleteobjects_module) + _FakeMinio.bucket_exists_value = True + _FakeMinio.objects = [] + _FakeMinio.removed_objects = [] + _FakeAdmin.calls = [] + _FakeAdmin.missing = False + + +def _config() -> MinIOProvisioningConfig: + return MinIOProvisioningConfig( + enabled=True, + endpoint="minio.local:9000", + public_endpoint="minio.local:9000", + admin_access_key="admin", + admin_secret_key="admin-secret", + bucket="beaver-user-files", + secure=False, + region=None, + ) + + +def _settings() -> MinIOSettings: + return MinIOSettings( + endpoint="minio.local:9000", + access_key="beaver-alice", + secret_key="alice-secret", + bucket="beaver-user-files", + namespace="users/alice", + ) + + +def _client(tmp_path, monkeypatch) -> TestClient: + monkeypatch.setenv("AUTHZ_DATA_DIR", str(tmp_path)) + monkeypatch.setenv("AUTHZ_PRIVATE_KEY_PATH", str(tmp_path / "signing_key.pem")) + monkeypatch.setenv("AUTHZ_INTERNAL_TOKEN", "test-internal-token") + monkeypatch.setenv("USER_FILES_MINIO_ENDPOINT", "minio.local:9000") + monkeypatch.setenv("USER_FILES_MINIO_ADMIN_ACCESS_KEY", "admin") + monkeypatch.setenv("USER_FILES_MINIO_ADMIN_SECRET_KEY", "admin-secret") + monkeypatch.setenv("USER_FILES_MINIO_BUCKET", "beaver-user-files") + import app.main as main + + main = importlib.reload(main) + return TestClient(main.app) + + +def _register_backend(client: TestClient) -> None: + response = client.post( + "/backends/register", + json={"backend_id": "alice", "name": "Alice", "base_url": "http://alice.local"}, + ) + assert response.status_code == 200 + + +def test_deprovision_removes_namespace_resources_without_secrets(monkeypatch) -> None: + _install_fake_minio(monkeypatch) + _FakeMinio.objects = [ + "users/alice/uploads/a.txt", + "users/alice/outputs/b.txt", + "users/bob/uploads/c.txt", + ] + + result = deprovision_user_file_minio_resources( + backend_id="alice", + existing=_settings(), + config=_config(), + ) + + assert result["ok"] is True + assert result["objects"] == {"status": "removed", "deleted": 2} + assert _FakeMinio.removed_objects == ["users/alice/uploads/a.txt", "users/alice/outputs/b.txt"] + assert ("user_remove", "beaver-alice") in _FakeAdmin.calls + assert ("policy_remove", "beaver-user-files-alice") in _FakeAdmin.calls + assert "secret" not in str(result).lower() + + +def test_deprovision_is_idempotent_when_resources_are_absent(monkeypatch) -> None: + _install_fake_minio(monkeypatch) + _FakeMinio.bucket_exists_value = False + _FakeAdmin.missing = True + + result = deprovision_user_file_minio_resources( + backend_id="alice", + existing=_settings(), + config=_config(), + ) + + assert result["ok"] is True + assert result["bucket"] == "absent" + assert result["objects"] == {"status": "absent", "deleted": 0} + assert result["user"] == {"status": "absent"} + assert result["policy"] == {"status": "absent"} + + +def test_internal_user_file_deprovision_requires_internal_token(tmp_path, monkeypatch) -> None: + with _client(tmp_path, monkeypatch) as client: + unauthorized = client.delete("/internal/backends/alice/user-files") + + assert unauthorized.status_code == 401 + + +def test_internal_user_file_deprovision_deletes_settings_without_returning_secret(tmp_path, monkeypatch) -> None: + _install_fake_minio(monkeypatch) + with _client(tmp_path, monkeypatch) as client: + _register_backend(client) + client.post( + "/backends/alice/settings/minio", + json={ + "endpoint": "minio.local:9000", + "access_key": "beaver-alice", + "secret_key": "alice-secret", + "bucket": "beaver-user-files", + "namespace": "users/alice", + }, + ) + deleted = client.delete( + "/internal/backends/alice/user-files", + headers={"Authorization": "Bearer test-internal-token"}, + ) + after_delete = client.get("/backends/alice/settings/minio") + + assert deleted.status_code == 200 + payload = deleted.json() + assert payload["ok"] is True + assert payload["settings"] == {"status": "removed"} + assert "secret" not in str(payload).lower() + assert after_delete.json() == {"configured": False} diff --git a/authz-service/src/tests/test_minio_settings.py b/authz-service/src/tests/test_minio_settings.py new file mode 100644 index 0000000..9b32d3d --- /dev/null +++ b/authz-service/src/tests/test_minio_settings.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import importlib + +from fastapi.testclient import TestClient + +from app.minio_provisioning import default_namespace, policy_json_for_backend + + +def _client(tmp_path, monkeypatch) -> TestClient: + monkeypatch.setenv("AUTHZ_DATA_DIR", str(tmp_path)) + monkeypatch.setenv("AUTHZ_PRIVATE_KEY_PATH", str(tmp_path / "signing_key.pem")) + monkeypatch.setenv("AUTHZ_INTERNAL_TOKEN", "test-internal-token") + import app.main as main + + main = importlib.reload(main) + return TestClient(main.app) + + +def _register_backend(client: TestClient) -> None: + response = client.post( + "/backends/register", + json={"backend_id": "alice", "name": "Alice", "base_url": "http://alice.local"}, + ) + assert response.status_code == 200 + + +def test_minio_settings_round_trip_with_masked_public_read(tmp_path, monkeypatch) -> None: + with _client(tmp_path, monkeypatch) as client: + _register_backend(client) + saved = client.post( + "/backends/alice/settings/minio", + json={ + "endpoint": "minio.local:9000", + "access_key": "alice-access", + "secret_key": "alice-secret", + "bucket": "beaver-user-files", + "namespace": "users/alice", + "secure": True, + "region": "us-east-1", + }, + ) + public = client.get("/backends/alice/settings/minio") + internal = client.get( + "/internal/backends/alice/settings/minio", + headers={"Authorization": "Bearer test-internal-token"}, + ) + + assert saved.status_code == 200 + assert saved.json()["configured"] is True + assert saved.json()["endpoint"] == "minio.local:9000" + assert saved.json()["secret_key_masked"] is True + assert "secret_key" not in saved.json() + assert public.status_code == 200 + assert public.json()["access_key"] == "alice-access" + assert public.json()["bucket"] == "beaver-user-files" + assert public.json()["namespace"] == "users/alice" + assert public.json()["secret_key_masked"] is True + assert "secret_key" not in public.json() + assert internal.status_code == 200 + assert internal.json()["secret_key"] == "alice-secret" + assert internal.json()["bucket"] == "beaver-user-files" + assert internal.json()["namespace"] == "users/alice" + + +def test_minio_settings_preserve_secret_on_masked_update(tmp_path, monkeypatch) -> None: + with _client(tmp_path, monkeypatch) as client: + _register_backend(client) + first = client.post( + "/backends/alice/settings/minio", + json={ + "endpoint": "minio.local:9000", + "access_key": "alice-access", + "secret_key": "alice-secret", + "bucket": "beaver-user-files", + "namespace": "users/alice", + }, + ) + updated = client.post( + "/backends/alice/settings/minio", + json={ + "endpoint": "minio2.local:9000", + "access_key": "alice-access-2", + "secret_key": "", + "bucket": "beaver-user-files", + "namespace": "users/alice-v2", + }, + ) + internal = client.get( + "/internal/backends/alice/settings/minio", + headers={"Authorization": "Bearer test-internal-token"}, + ) + + assert first.status_code == 200 + assert updated.status_code == 200 + assert updated.json()["endpoint"] == "minio2.local:9000" + assert updated.json()["namespace"] == "users/alice-v2" + assert internal.status_code == 200 + assert internal.json()["secret_key"] == "alice-secret" + assert internal.json()["bucket"] == "beaver-user-files" + assert internal.json()["namespace"] == "users/alice-v2" + + +def test_minio_settings_delete_and_missing_behavior(tmp_path, monkeypatch) -> None: + with _client(tmp_path, monkeypatch) as client: + _register_backend(client) + missing_public = client.get("/backends/alice/settings/minio") + missing_internal = client.get( + "/internal/backends/alice/settings/minio", + headers={"Authorization": "Bearer test-internal-token"}, + ) + client.post( + "/backends/alice/settings/minio", + json={ + "endpoint": "minio.local:9000", + "access_key": "alice-access", + "secret_key": "alice-secret", + "bucket": "beaver-user-files", + "namespace": "users/alice", + }, + ) + deleted = client.delete("/backends/alice/settings/minio") + after_delete = client.get("/backends/alice/settings/minio") + + assert missing_public.status_code == 200 + assert missing_public.json() == {"configured": False} + assert missing_internal.status_code == 404 + assert deleted.status_code == 200 + assert deleted.json() == {"ok": True} + assert after_delete.status_code == 200 + assert after_delete.json() == {"configured": False} + + +def test_minio_namespace_policy_is_scoped_to_backend_prefix() -> None: + policy = policy_json_for_backend(backend_id="alice", bucket="beaver-user-files") + + assert default_namespace("alice") == "users/alice" + assert "arn:aws:s3:::beaver-user-files/users/alice/*" in policy + assert "users/bob" not in policy diff --git a/authz-service/src/uv.lock b/authz-service/src/uv.lock index 3c80b48..b63ad02 100644 --- a/authz-service/src/uv.lock +++ b/authz-service/src/uv.lock @@ -1,6 +1,10 @@ version = 1 revision = 3 requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version < '3.14'", +] [[package]] name = "annotated-doc" @@ -34,6 +38,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/3c0a35f46e52108d4707c44b95cfe2afcafc50800b5450c197454569b776/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:3d3f05610594151994ca9ccb3c771115bdb4daef161976a266f0dd8aa9996b8f", size = 54393, upload-time = "2025-07-30T10:01:40.97Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/98bbd6ee89febd4f212696f13c03ca302b8552e7dbf9c8efa11ea4a388c3/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8b8efee945193e667a396cbc7b4fb7d357297d6234d30a489905d96caabde56b", size = 29328, upload-time = "2025-07-30T10:01:41.916Z" }, + { url = "https://files.pythonhosted.org/packages/43/24/90a01c0ef12ac91a6be05969f29944643bc1e5e461155ae6559befa8f00b/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3c6702abc36bf3ccba3f802b799505def420a1b7039862014a65db3205967f5a", size = 31269, upload-time = "2025-07-30T10:01:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d3/942aa10782b2697eee7af5e12eeff5ebb325ccfb86dd8abda54174e377e4/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44", size = 86558, upload-time = "2025-07-30T10:01:43.943Z" }, + { url = "https://files.pythonhosted.org/packages/0d/82/b484f702fec5536e71836fc2dbc8c5267b3f6e78d2d539b4eaa6f0db8bf8/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb", size = 92364, upload-time = "2025-07-30T10:01:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c1/a606ff83b3f1735f3759ad0f2cd9e038a0ad11a3de3b6c673aa41c24bb7b/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92", size = 85637, upload-time = "2025-07-30T10:01:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/44/b4/678503f12aceb0262f84fa201f6027ed77d71c5019ae03b399b97caa2f19/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85", size = 91934, upload-time = "2025-07-30T10:01:47.203Z" }, + { url = "https://files.pythonhosted.org/packages/f0/c7/f36bd08ef9bd9f0a9cff9428406651f5937ce27b6c5b07b92d41f91ae541/argon2_cffi_bindings-25.1.0-cp314-cp314t-win32.whl", hash = "sha256:84a461d4d84ae1295871329b346a97f68eade8c53b6ed9a7ca2d7467f3c8ff6f", size = 28158, upload-time = "2025-07-30T10:01:48.341Z" }, + { url = "https://files.pythonhosted.org/packages/b3/80/0106a7448abb24a2c467bf7d527fe5413b7fdfa4ad6d6a96a43a62ef3988/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b55aec3565b65f56455eebc9b9f34130440404f27fe21c3b375bf1ea4d8fbae6", size = 32597, upload-time = "2025-07-30T10:01:49.112Z" }, + { url = "https://files.pythonhosted.org/packages/05/b8/d663c9caea07e9180b2cb662772865230715cbd573ba3b5e81793d580316/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:87c33a52407e4c41f3b70a9c2d3f6056d88b10dad7695be708c5021673f55623", size = 28231, upload-time = "2025-07-30T10:01:49.92Z" }, + { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121, upload-time = "2025-07-30T10:01:50.815Z" }, + { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177, upload-time = "2025-07-30T10:01:51.681Z" }, + { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090, upload-time = "2025-07-30T10:01:53.184Z" }, + { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, + { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, + { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, + { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180, upload-time = "2025-07-30T10:01:57.759Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715, upload-time = "2025-07-30T10:01:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149, upload-time = "2025-07-30T10:01:59.329Z" }, + { url = "https://files.pythonhosted.org/packages/11/2d/ba4e4ca8d149f8dcc0d952ac0967089e1d759c7e5fcf0865a317eb680fbb/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:6dca33a9859abf613e22733131fc9194091c1fa7cb3e131c143056b4856aa47e", size = 24549, upload-time = "2025-07-30T10:02:00.101Z" }, + { url = "https://files.pythonhosted.org/packages/5c/82/9b2386cc75ac0bd3210e12a44bfc7fd1632065ed8b80d573036eecb10442/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:21378b40e1b8d1655dd5310c84a40fc19a9aa5e6366e835ceb8576bf0fea716d", size = 25539, upload-time = "2025-07-30T10:02:00.929Z" }, + { url = "https://files.pythonhosted.org/packages/31/db/740de99a37aa727623730c90d92c22c9e12585b3c98c54b7960f7810289f/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d588dec224e2a83edbdc785a5e6f3c6cd736f46bfd4b441bbb5aa1f5085e584", size = 28467, upload-time = "2025-07-30T10:02:02.08Z" }, + { url = "https://files.pythonhosted.org/packages/71/7a/47c4509ea18d755f44e2b92b7178914f0c113946d11e16e626df8eaa2b0b/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5acb4e41090d53f17ca1110c3427f0a130f944b896fc8c83973219c97f57b690", size = 27355, upload-time = "2025-07-30T10:02:02.867Z" }, + { url = "https://files.pythonhosted.org/packages/ee/82/82745642d3c46e7cea25e1885b014b033f4693346ce46b7f47483cf5d448/argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:da0c79c23a63723aa5d782250fbf51b768abca630285262fb5144ba5ae01e520", size = 29187, upload-time = "2025-07-30T10:02:03.674Z" }, +] + [[package]] name = "authz-service" version = "0.1.0" @@ -42,6 +94,7 @@ dependencies = [ { name = "cryptography" }, { name = "fastapi" }, { name = "httpx" }, + { name = "minio" }, { name = "pydantic" }, { name = "pyjwt" }, { name = "python-multipart" }, @@ -58,6 +111,7 @@ requires-dist = [ { name = "cryptography", specifier = ">=45.0.0,<46.0.0" }, { name = "fastapi", specifier = ">=0.115.0,<1.0.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0.0" }, + { name = "minio", specifier = ">=7.2.0,<8.0.0" }, { name = "pydantic", specifier = ">=2.12.0,<3.0.0" }, { name = "pyjwt", specifier = ">=2.10.0,<3.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0,<9.0.0" }, @@ -351,6 +405,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "minio" +version = "7.2.20" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/df/6dfc6540f96a74125a11653cce717603fd5b7d0001a8e847b3e54e72d238/minio-7.2.20.tar.gz", hash = "sha256:95898b7a023fbbfde375985aa77e2cd6a0762268db79cf886f002a9ea8e68598", size = 136113, upload-time = "2025-11-27T00:37:15.569Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -378,6 +448,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" }, + { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" }, + { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" }, + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, + { url = "https://files.pythonhosted.org/packages/d9/12/e33935a0709c07de084d7d58d330ec3f4daf7910a18e77937affdb728452/pycryptodome-3.23.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ddb95b49df036ddd264a0ad246d1be5b672000f12d6961ea2c267083a5e19379", size = 1623886, upload-time = "2025-05-17T17:21:20.614Z" }, + { url = "https://files.pythonhosted.org/packages/22/0b/aa8f9419f25870889bebf0b26b223c6986652bdf071f000623df11212c90/pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e95564beb8782abfd9e431c974e14563a794a4944c29d6d3b7b5ea042110b4", size = 1672151, upload-time = "2025-05-17T17:21:22.666Z" }, + { url = "https://files.pythonhosted.org/packages/d4/5e/63f5cbde2342b7f70a39e591dbe75d9809d6338ce0b07c10406f1a140cdc/pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14e15c081e912c4b0d75632acd8382dfce45b258667aa3c67caf7a4d4c13f630", size = 1664461, upload-time = "2025-05-17T17:21:25.225Z" }, + { url = "https://files.pythonhosted.org/packages/d6/92/608fbdad566ebe499297a86aae5f2a5263818ceeecd16733006f1600403c/pycryptodome-3.23.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fc76bf273353dc7e5207d172b83f569540fc9a28d63171061c42e361d22353", size = 1702440, upload-time = "2025-05-17T17:21:27.991Z" }, + { url = "https://files.pythonhosted.org/packages/d1/92/2eadd1341abd2989cce2e2740b4423608ee2014acb8110438244ee97d7ff/pycryptodome-3.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:45c69ad715ca1a94f778215a11e66b7ff989d792a4d63b68dc586a1da1392ff5", size = 1803005, upload-time = "2025-05-17T17:21:31.37Z" }, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -717,6 +822,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + [[package]] name = "uvicorn" version = "0.41.0" diff --git a/deploy-control/.env.example b/deploy-control/.env.example index 637795e..3d5f8a9 100644 --- a/deploy-control/.env.example +++ b/deploy-control/.env.example @@ -18,7 +18,7 @@ DEFAULT_AUTHZ_OUTLOOK_MCP_URL= DEFAULT_OUTLOOK_MCP_SERVER_ID=outlook_mcp DEPLOY_PUBLIC_SCHEME=http -DEPLOY_PUBLIC_BASE_DOMAIN=203.0.113.10.nip.io +DEPLOY_PUBLIC_BASE_DOMAIN=localhost DEPLOY_PUBLIC_HOST_TEMPLATE={slug}.{base_domain} DEPLOY_PUBLIC_PORT=8088 DEPLOY_AUTO_START_PROXY=1 @@ -26,5 +26,5 @@ DEPLOY_HEALTH_TIMEOUT_SECONDS=60 DEPLOY_HEALTH_INTERVAL_SECONDS=1 # Passed through to create-instance.sh when the app-instance image is rebuilt. -AUTH_PORTAL_URL=http://203.0.113.10:3081 +AUTH_PORTAL_URL=http://127.0.0.1:3081 AUTH_PORTAL_PORT=3081 diff --git a/deploy-control/README.md b/deploy-control/README.md index 55ab213..6766075 100644 --- a/deploy-control/README.md +++ b/deploy-control/README.md @@ -32,7 +32,7 @@ 默认实例 URL 形如： ```text -http://.127.0.0.1.nip.io:8088 +http://.localhost:8088 ``` 实例容器本身的 `20000-29999` 端口默认只绑定到部署机 `127.0.0.1`，外部入口应走 `router-proxy`。 diff --git a/deploy-control/server.py b/deploy-control/server.py index 9e0dfad..86aded9 100755 --- a/deploy-control/server.py +++ b/deploy-control/server.py @@ -11,6 +11,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Any from urllib import error as urllib_error +from urllib import parse as urllib_parse from urllib import request as urllib_request @@ -37,15 +38,18 @@ API_TOKEN = os.environ.get("DEPLOY_CONTROL_API_TOKEN", "").strip() INSTANCE_IMAGE = os.environ.get("APP_INSTANCE_IMAGE", "beaver/app-instance:latest").strip() INSTANCE_NETWORK_NAME = os.environ.get("APP_INSTANCE_NETWORK_NAME", "beaver-instance-edge").strip() DEFAULT_AUTHZ_BASE_URL = os.environ.get("DEFAULT_AUTHZ_BASE_URL", "").strip() +DEFAULT_AUTHZ_INTERNAL_TOKEN = os.environ.get("DEFAULT_AUTHZ_INTERNAL_TOKEN", "").strip() DEFAULT_AUTHZ_OUTLOOK_MCP_URL = os.environ.get("DEFAULT_AUTHZ_OUTLOOK_MCP_URL", "").strip() DEFAULT_OUTLOOK_MCP_SERVER_ID = os.environ.get("DEFAULT_OUTLOOK_MCP_SERVER_ID", "outlook_mcp").strip() or "outlook_mcp" +DEFAULT_USER_FILES_MAX_UPLOAD_BYTES = os.environ.get("DEFAULT_USER_FILES_MAX_UPLOAD_BYTES", "").strip() PUBLIC_SCHEME = os.environ.get("DEPLOY_PUBLIC_SCHEME", "http").strip() or "http" -PUBLIC_BASE_DOMAIN = os.environ.get("DEPLOY_PUBLIC_BASE_DOMAIN", "127.0.0.1.nip.io").strip() +PUBLIC_BASE_DOMAIN = os.environ.get("DEPLOY_PUBLIC_BASE_DOMAIN", "localhost").strip() PUBLIC_HOST_TEMPLATE = os.environ.get("DEPLOY_PUBLIC_HOST_TEMPLATE", "{slug}.{base_domain}").strip() PUBLIC_PORT = int(os.environ.get("DEPLOY_PUBLIC_PORT", "8088").strip() or "8088") AUTO_START_PROXY = os.environ.get("DEPLOY_AUTO_START_PROXY", "1").strip() not in {"0", "false", "False"} HEALTH_TIMEOUT_SECONDS = float(os.environ.get("DEPLOY_HEALTH_TIMEOUT_SECONDS", "60").strip() or "60") HEALTH_INTERVAL_SECONDS = float(os.environ.get("DEPLOY_HEALTH_INTERVAL_SECONDS", "1").strip() or "1") +UPSTREAM_TIMEOUT_SECONDS = float(os.environ.get("DEPLOY_UPSTREAM_TIMEOUT_SECONDS", "90").strip() or "90") INSTANCE_INTERNAL_PORT = int(os.environ.get("APP_INSTANCE_INTERNAL_PORT", "8080").strip() or "8080") SERVER_HOST = os.environ.get("DEPLOY_CONTROL_HOST", "0.0.0.0").strip() or "0.0.0.0" SERVER_PORT = int(os.environ.get("DEPLOY_CONTROL_PORT", "8090").strip() or "8090") @@ -263,9 +267,13 @@ def create_or_get_instance(payload: dict[str, Any]) -> dict[str, Any]: ] if authz_base_url: command.extend(["--authz-base-url", authz_base_url]) + if DEFAULT_AUTHZ_INTERNAL_TOKEN: + command.extend(["--authz-internal-token", DEFAULT_AUTHZ_INTERNAL_TOKEN]) if authz_outlook_mcp_url: command.extend(["--authz-outlook-mcp-url", authz_outlook_mcp_url]) command.extend(["--outlook-mcp-server-id", DEFAULT_OUTLOOK_MCP_SERVER_ID]) + if DEFAULT_USER_FILES_MAX_UPLOAD_BYTES: + command.extend(["--user-files-max-upload-bytes", DEFAULT_USER_FILES_MAX_UPLOAD_BYTES]) if payload.get("replace") is True: command.append("--replace") @@ -498,21 +506,108 @@ def resolve_instance(payload: dict[str, Any]) -> dict[str, Any]: } -def remove_instance(instance_id: str, purge_data: bool) -> dict[str, Any]: +def deprovision_user_files( + *, + backend_id: str, + authz_base_url: str, + best_effort: bool = True, +) -> dict[str, Any]: + if not backend_id: + return {"ok": False, "status": "failed", "error": "backend_id is missing"} + if not authz_base_url: + return {"ok": False, "status": "failed", "error": "AuthZ base URL is not configured"} + if not DEFAULT_AUTHZ_INTERNAL_TOKEN: + return {"ok": False, "status": "failed", "error": "AuthZ internal token is not configured"} + + query = urllib_parse.urlencode({"best_effort": "1" if best_effort else "0"}) + quoted_backend_id = urllib_parse.quote(backend_id, safe="") + url = f"{authz_base_url.rstrip('/')}/internal/backends/{quoted_backend_id}/user-files?{query}" + request = urllib_request.Request( + url, + method="DELETE", + headers={ + "Authorization": f"Bearer {DEFAULT_AUTHZ_INTERNAL_TOKEN}", + "Accept": "application/json", + }, + ) + try: + with urllib_request.urlopen(request, timeout=UPSTREAM_TIMEOUT_SECONDS) as response: + raw = response.read().decode("utf-8") + except urllib_error.HTTPError as exc: + detail = exc.reason + try: + payload = json.loads(exc.read().decode("utf-8")) + if isinstance(payload, dict): + detail = str(payload.get("detail") or detail) + except Exception: + pass + return {"ok": False, "status": "failed", "error": detail, "status_code": exc.code} + except (urllib_error.URLError, TimeoutError) as exc: + return {"ok": False, "status": "failed", "error": str(exc)} + + if not raw.strip(): + return {"ok": True, "status": "removed"} + try: + payload = json.loads(raw) + except json.JSONDecodeError: + return {"ok": False, "status": "failed", "error": "AuthZ response was not valid JSON"} + if not isinstance(payload, dict): + return {"ok": False, "status": "failed", "error": "AuthZ response must be a JSON object"} + payload.setdefault("ok", True) + return payload + + +def remove_instance(instance_id: str, purge_data: bool, purge_user_files: bool = False) -> dict[str, Any]: if not instance_id.strip(): raise ApiError(HTTPStatus.BAD_REQUEST, "instance id is required") + record = get_registry_record(instance_id=instance_id) + if record is None: + local_result: dict[str, Any] = { + "instance_id": instance_id, + "status": "already_absent", + "already_absent": True, + } + user_files_result: dict[str, Any] = {"ok": True, "status": "skipped"} + if purge_user_files: + user_files_result = deprovision_user_files( + backend_id=instance_id, + authz_base_url=DEFAULT_AUTHZ_BASE_URL, + best_effort=True, + ) + return { + "ok": not purge_user_files or bool(user_files_result.get("ok")), + "instance": local_result, + "local": local_result, + "user_files": user_files_result, + } + backend_id = str(record.get("backend_id", "") or record.get("username", "") or instance_id).strip() + authz_base_url = str(record.get("authz_base_url", "") or DEFAULT_AUTHZ_BASE_URL).strip() + command = [str(REMOVE_INSTANCE_SCRIPT), "--instance-id", instance_id] if purge_data: command.append("--purge-data") output = run_command(command, cwd=APP_INSTANCE_DIR) ensure_proxy() - result: dict[str, str] = {} + local_result: dict[str, str] = {} for line in output.splitlines(): if "=" not in line: continue key, value = line.split("=", 1) - result[key] = value - return result + local_result[key] = value + + user_files_result: dict[str, Any] = {"ok": True, "status": "skipped"} + if purge_user_files: + user_files_result = deprovision_user_files( + backend_id=backend_id, + authz_base_url=authz_base_url, + best_effort=True, + ) + return { + "ok": bool(local_result) and (not purge_user_files or bool(user_files_result.get("ok"))), + "instance": local_result, + "local": local_result, + "user_files": user_files_result, + } class Handler(BaseHTTPRequestHandler): @@ -587,11 +682,17 @@ class Handler(BaseHTTPRequestHandler): def do_DELETE(self) -> None: # noqa: N802 try: self._require_auth() - if not self.path.startswith("/api/instances/"): + parsed = urllib_parse.urlparse(self.path) + if not parsed.path.startswith("/api/instances/"): raise ApiError(HTTPStatus.NOT_FOUND, "not found") - instance_id = self.path.rsplit("/", 1)[-1] + instance_id = urllib_parse.unquote(parsed.path.rsplit("/", 1)[-1]) + query = urllib_parse.parse_qs(parsed.query) purge_data = self.headers.get("X-Purge-Data", "").strip() == "1" - self._json_response(HTTPStatus.OK, remove_instance(instance_id, purge_data)) + purge_user_files = ( + self.headers.get("X-Purge-User-Files", "").strip() == "1" + or query.get("purge_user_files", [""])[-1] in {"1", "true", "True", "yes"} + ) + self._json_response(HTTPStatus.OK, remove_instance(instance_id, purge_data, purge_user_files)) except ApiError as exc: self._json_response(exc.status_code, {"detail": exc.detail}) diff --git a/deploy-control/tests/test_delete_orchestration.py b/deploy-control/tests/test_delete_orchestration.py new file mode 100644 index 0000000..4aab7c9 --- /dev/null +++ b/deploy-control/tests/test_delete_orchestration.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + + +SERVER_PATH = Path(__file__).resolve().parents[1] / "server.py" + + +def _load_server_module(): + spec = importlib.util.spec_from_file_location("deploy_control_server_for_tests", SERVER_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _record() -> dict[str, Any]: + return { + "instance_id": "smoke-user", + "username": "smoke-user", + "backend_id": "smoke-backend", + "authz_base_url": "http://authz.local", + } + + +def test_remove_instance_without_user_file_purge_skips_authz(monkeypatch) -> None: + server = _load_server_module() + calls: list[Any] = [] + + monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: _record()) + monkeypatch.setattr( + server, + "run_command", + lambda *_args, **_kwargs: "instance_id=smoke-user\npurged_data=1", + ) + monkeypatch.setattr(server, "ensure_proxy", lambda: None) + monkeypatch.setattr(server, "deprovision_user_files", lambda **kwargs: calls.append(kwargs)) + + result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=False) + + assert result["ok"] is True + assert result["local"]["instance_id"] == "smoke-user" + assert result["user_files"] == {"ok": True, "status": "skipped"} + assert calls == [] + + +def test_remove_instance_with_user_file_purge_calls_authz_after_resolving_backend(monkeypatch) -> None: + server = _load_server_module() + calls: list[dict[str, Any]] = [] + + monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: _record()) + monkeypatch.setattr( + server, + "run_command", + lambda *_args, **_kwargs: "instance_id=smoke-user\npurged_data=1", + ) + monkeypatch.setattr(server, "ensure_proxy", lambda: None) + + def fake_deprovision(**kwargs: Any) -> dict[str, Any]: + calls.append(kwargs) + return {"ok": True, "objects": {"status": "removed", "deleted": 1}} + + monkeypatch.setattr(server, "deprovision_user_files", fake_deprovision) + + result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=True) + + assert result["ok"] is True + assert calls == [ + { + "backend_id": "smoke-backend", + "authz_base_url": "http://authz.local", + "best_effort": True, + } + ] + assert result["user_files"]["objects"] == {"status": "removed", "deleted": 1} + + +def test_remove_instance_reports_user_file_cleanup_failure_separately(monkeypatch) -> None: + server = _load_server_module() + + monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: _record()) + monkeypatch.setattr( + server, + "run_command", + lambda *_args, **_kwargs: "instance_id=smoke-user\npurged_data=1", + ) + monkeypatch.setattr(server, "ensure_proxy", lambda: None) + monkeypatch.setattr( + server, + "deprovision_user_files", + lambda **_kwargs: {"ok": False, "status": "failed", "error": "AuthZ unavailable"}, + ) + + result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=True) + + assert result["ok"] is False + assert result["local"]["instance_id"] == "smoke-user" + assert result["user_files"] == {"ok": False, "status": "failed", "error": "AuthZ unavailable"} + + +def test_remove_already_absent_instance_is_idempotent_without_file_purge(monkeypatch) -> None: + server = _load_server_module() + calls: list[Any] = [] + + monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: None) + monkeypatch.setattr(server, "run_command", lambda *_args, **_kwargs: calls.append(_args)) + monkeypatch.setattr(server, "deprovision_user_files", lambda **kwargs: calls.append(kwargs)) + + result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=False) + + assert result["ok"] is True + assert result["local"] == { + "instance_id": "smoke-user", + "status": "already_absent", + "already_absent": True, + } + assert result["user_files"] == {"ok": True, "status": "skipped"} + assert calls == [] + + +def test_remove_already_absent_instance_can_retry_user_file_cleanup(monkeypatch) -> None: + server = _load_server_module() + calls: list[dict[str, Any]] = [] + monkeypatch.setattr(server, "DEFAULT_AUTHZ_BASE_URL", "http://authz.local") + + monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: None) + monkeypatch.setattr(server, "run_command", lambda *_args, **_kwargs: "should-not-run") + + def fake_deprovision(**kwargs: Any) -> dict[str, Any]: + calls.append(kwargs) + return {"ok": True, "settings_found": False, "objects": {"status": "absent"}} + + monkeypatch.setattr(server, "deprovision_user_files", fake_deprovision) + + result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=True) + + assert result["ok"] is True + assert result["local"]["already_absent"] is True + assert calls == [ + { + "backend_id": "smoke-user", + "authz_base_url": "http://authz.local", + "best_effort": True, + } + ] + assert result["user_files"]["settings_found"] is False diff --git a/router-proxy/README.md b/router-proxy/README.md index ca949a1..aa7a182 100644 --- a/router-proxy/README.md +++ b/router-proxy/README.md @@ -49,7 +49,7 @@ cd /home/ivan/xuan/beaver_project/router-proxy 如果 deploy-control 侧使用默认配置，实例 URL 形如： ```text -http://.127.0.0.1.nip.io:8088 +http://.localhost:8088 ``` 只要本机或 DNS 能把该域名解析到代理所在机器，就会由该代理转发到目标实例容器。 diff --git a/router-proxy/nginx.conf b/router-proxy/nginx.conf index c15e630..88976df 100644 --- a/router-proxy/nginx.conf +++ b/router-proxy/nginx.conf @@ -12,7 +12,7 @@ http { tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; - client_max_body_size 100m; + client_max_body_size 5g; map $http_upgrade $connection_upgrade { default upgrade; diff --git a/scripts/check-minio-prefix-policy.py b/scripts/check-minio-prefix-policy.py new file mode 100755 index 0000000..f9ff707 --- /dev/null +++ b/scripts/check-minio-prefix-policy.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Check that a provisioned MinIO user can access only its own prefix.""" + +from __future__ import annotations + +import os +import sys +from io import BytesIO + +from minio import Minio +from minio.error import S3Error + + +def main() -> int: + endpoint = _require("BEAVER_CHECK_MINIO_ENDPOINT") + bucket = os.getenv("BEAVER_CHECK_MINIO_BUCKET", "beaver-user-files").strip() + access_key = _require("BEAVER_CHECK_MINIO_ACCESS_KEY") + secret_key = _require("BEAVER_CHECK_MINIO_SECRET_KEY") + own_backend = _require("BEAVER_CHECK_MINIO_BACKEND_ID") + other_backend = os.getenv("BEAVER_CHECK_MINIO_OTHER_BACKEND_ID", "policy-denied-other").strip() + secure = os.getenv("BEAVER_CHECK_MINIO_SECURE", "0").strip().lower() in {"1", "true", "yes", "on"} + + client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure) + own_object = f"users/{own_backend}/uploads/policy-check.txt" + other_object = f"users/{other_backend}/uploads/policy-check.txt" + + client.put_object(bucket, own_object, BytesIO(b"ok"), length=2, content_type="text/plain") + client.stat_object(bucket, own_object) + + try: + client.put_object(bucket, other_object, BytesIO(b"no"), length=2, content_type="text/plain") + except S3Error as exc: + if exc.code in {"AccessDenied", "AccessDeniedException"}: + print(f"ok: {access_key} can access {own_object} and is denied for {other_object}") + return 0 + raise + + print(f"error: {access_key} unexpectedly wrote {other_object}", file=sys.stderr) + return 1 + + +def _require(name: str) -> str: + value = os.getenv(name, "").strip() + if not value: + raise SystemExit(f"{name} is required") + return value + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/cleanup-test-users.py b/scripts/cleanup-test-users.py new file mode 100755 index 0000000..9703a7b --- /dev/null +++ b/scripts/cleanup-test-users.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any +from urllib import error as urllib_error +from urllib import parse as urllib_parse +from urllib import request as urllib_request + + +ROOT_DIR = Path(__file__).resolve().parents[1] +DEFAULT_REGISTRY = ROOT_DIR / "app-instance" / "runtime" / "registry" / "instances.json" +SAFE_PREFIXES = ("smoke", "test", "debug") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Dry-run or purge Beaver smoke/test users.") + parser.add_argument("--registry", default=str(DEFAULT_REGISTRY), help="Path to app-instance registry JSON.") + parser.add_argument("--deploy-control-url", default=os.getenv("DEPLOY_CONTROL_URL", "http://127.0.0.1:8090")) + parser.add_argument("--token", default=os.getenv("DEPLOY_CONTROL_API_TOKEN", "")) + parser.add_argument("--backend-id", action="append", default=[], help="Explicit backend id to clean.") + parser.add_argument("--instance-id", action="append", default=[], help="Explicit instance id to clean.") + parser.add_argument("--username-prefix", action="append", default=[], help="Safe test username prefix, e.g. smoke.") + parser.add_argument("--execute", action="store_true", help="Actually delete matched instances.") + parser.add_argument("--purge-data", action="store_true", help="Delete local instance data when executing.") + parser.add_argument( + "--keep-user-files", + action="store_true", + help="Do not request MinIO/user-file purge when executing.", + ) + parser.add_argument( + "--allow-any-prefix", + action="store_true", + help="Allow non-test username prefixes. Use only for controlled maintenance.", + ) + return parser.parse_args() + + +def load_registry(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + payload = json.loads(path.read_text(encoding="utf-8")) + instances = payload.get("instances", []) if isinstance(payload, dict) else [] + return [item for item in instances if isinstance(item, dict)] + + +def selected_instances(args: argparse.Namespace) -> list[dict[str, Any]]: + instances = load_registry(Path(args.registry).expanduser()) + explicit_backend_ids = {item.strip() for item in args.backend_id if item.strip()} + explicit_instance_ids = {item.strip() for item in args.instance_id if item.strip()} + prefixes = tuple(item.strip() for item in args.username_prefix if item.strip()) + + if not explicit_backend_ids and not explicit_instance_ids and not prefixes: + raise SystemExit("Refusing cleanup: provide --backend-id, --instance-id, or --username-prefix.") + unsafe = [prefix for prefix in prefixes if not prefix.startswith(SAFE_PREFIXES)] + if unsafe and not args.allow_any_prefix: + raise SystemExit( + "Refusing cleanup: username prefixes must start with " + f"{', '.join(SAFE_PREFIXES)} unless --allow-any-prefix is set." + ) + + matches: list[dict[str, Any]] = [] + for item in instances: + instance_id = str(item.get("instance_id", "") or "") + backend_id = str(item.get("backend_id", "") or "") + username = str(item.get("username", "") or "") + if instance_id in explicit_instance_ids or backend_id in explicit_backend_ids: + matches.append(item) + continue + if prefixes and any(username.startswith(prefix) or backend_id.startswith(prefix) for prefix in prefixes): + matches.append(item) + return matches + + +def delete_instance(args: argparse.Namespace, instance_id: str) -> dict[str, Any]: + base_url = args.deploy_control_url.rstrip("/") + url = f"{base_url}/api/instances/{urllib_parse.quote(instance_id, safe='')}" + headers = {"Accept": "application/json"} + if args.token.strip(): + headers["Authorization"] = f"Bearer {args.token.strip()}" + if args.purge_data: + headers["X-Purge-Data"] = "1" + if not args.keep_user_files: + headers["X-Purge-User-Files"] = "1" + + request = urllib_request.Request(url, method="DELETE", headers=headers) + try: + with urllib_request.urlopen(request, timeout=120) as response: + raw = response.read().decode("utf-8") + except urllib_error.HTTPError as exc: + detail = exc.reason + try: + payload = json.loads(exc.read().decode("utf-8")) + if isinstance(payload, dict): + detail = str(payload.get("detail") or detail) + except Exception: + pass + return {"ok": False, "instance_id": instance_id, "error": detail, "status_code": exc.code} + except urllib_error.URLError as exc: + return {"ok": False, "instance_id": instance_id, "error": str(exc)} + if not raw.strip(): + return {"ok": True, "instance_id": instance_id} + try: + payload = json.loads(raw) + except json.JSONDecodeError: + return {"ok": False, "instance_id": instance_id, "error": "deploy-control response was not valid JSON"} + return payload if isinstance(payload, dict) else {"ok": False, "instance_id": instance_id, "error": "unexpected response"} + + +def main() -> int: + args = parse_args() + matches = selected_instances(args) + planned = [ + { + "instance_id": str(item.get("instance_id", "") or ""), + "backend_id": str(item.get("backend_id", "") or ""), + "username": str(item.get("username", "") or ""), + "instance_root": str(item.get("instance_root", "") or ""), + } + for item in matches + ] + if not args.execute: + print(json.dumps({"dry_run": True, "count": len(planned), "planned": planned}, indent=2, ensure_ascii=False)) + return 0 + + results = [] + for item in planned: + instance_id = item["instance_id"] + if not instance_id: + results.append({"ok": False, "error": "matched registry record is missing instance_id", "record": item}) + continue + results.append(delete_instance(args, instance_id)) + ok = all(bool(item.get("ok")) for item in results) + print(json.dumps({"dry_run": False, "count": len(results), "ok": ok, "results": results}, indent=2, ensure_ascii=False)) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke-auth-files.mjs b/scripts/smoke-auth-files.mjs new file mode 100755 index 0000000..aa684ba --- /dev/null +++ b/scripts/smoke-auth-files.mjs @@ -0,0 +1,190 @@ +#!/usr/bin/env node +import { createRequire } from 'node:module'; +import { execFileSync } from 'node:child_process'; + +const require = createRequire(import.meta.url); +const { chromium } = require('playwright'); + +const now = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); +const username = process.env.BEAVER_SMOKE_USERNAME || `smoke${now}`; +const password = process.env.BEAVER_SMOKE_PASSWORD || 'TestPass123!'; +const email = process.env.BEAVER_SMOKE_EMAIL || `${username}@example.test`; +const portalUrl = (process.env.BEAVER_SMOKE_PORTAL_URL || 'http://127.0.0.1:3081').replace(/\/$/, ''); +const nextPath = process.env.BEAVER_SMOKE_NEXT_PATH || '/files'; +const executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined; +const hostResolverRules = process.env.BEAVER_SMOKE_HOST_RESOLVER_RULES || ''; +const headless = process.env.BEAVER_SMOKE_HEADLESS !== '0'; +const verifyMinio = process.env.BEAVER_SMOKE_VERIFY_MINIO === '1'; + +const launchArgs = []; +if (hostResolverRules) { + launchArgs.push(`--host-resolver-rules=${hostResolverRules}`); +} + +const failedResponses = []; +const failedRequests = []; + +function remember(list, value) { + list.push(value); + if (list.length > 30) { + list.shift(); + } +} + +async function clickFirstVisible(locators) { + for (const locator of locators) { + if (await locator.first().isVisible().catch(() => false)) { + await locator.first().click(); + return true; + } + } + return false; +} + +async function main() { + const browser = await chromium.launch({ + headless, + executablePath, + args: launchArgs, + }); + const page = await browser.newPage({ viewport: { width: 1365, height: 900 } }); + + page.on('response', (response) => { + if (response.status() >= 400) { + remember(failedResponses, `${response.status()} ${response.url()}`); + } + }); + page.on('requestfailed', (request) => { + remember(failedRequests, `${request.url()} ${request.failure()?.errorText || ''}`); + }); + + try { + const registerUrl = new URL('/register', portalUrl); + registerUrl.searchParams.set('next', nextPath); + await page.goto(registerUrl.toString(), { waitUntil: 'domcontentloaded', timeout: 30000 }); + + await page.locator('#username').fill(username); + await page.locator('#email').fill(email); + await page.locator('#password').fill(password); + await page.locator('#confirmPassword').fill(password); + const submitButton = page.locator('form').first().locator('button[type="submit"]'); + await submitButton.waitFor({ state: 'visible', timeout: 30000 }); + await submitButton.click(); + await page.waitForResponse( + (response) => response.url().includes('/api/runtime/register'), + { timeout: 180000 }, + ); + + await page.getByText(/配置模型|Model Setup/).waitFor({ timeout: 180000 }); + const skipped = await clickFirstVisible([ + page.getByRole('button', { name: /跳过|Skip/i }), + page.locator('button.secondary-button'), + ]); + if (!skipped) { + throw new Error('Model setup skip button was not found'); + } + + await page.waitForURL(new RegExp(`${nextPath.replace('/', '\\/')}$`), { timeout: 60000 }); + await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {}); + await page.getByText(/文件管理|Files/).waitFor({ timeout: 30000 }); + + for (const root of ['uploads', 'outputs', 'shared', 'tasks']) { + await page.getByText(root, { exact: true }).waitFor({ timeout: 30000 }); + } + + const token = await page.evaluate(() => window.localStorage.getItem('beaver_access_token') || ''); + if (!token) { + throw new Error('No Beaver access token found after handoff'); + } + const uploadName = `smoke-${Date.now()}.txt`; + const uploadBody = `hello from Beaver smoke ${username}`; + const uploadResult = await page.evaluate(async ({ token, uploadName, uploadBody }) => { + const form = new FormData(); + form.set('path', 'uploads'); + form.set('file', new File([uploadBody], uploadName, { type: 'text/plain' })); + const response = await fetch('/api/user-files/upload', { + method: 'POST', + headers: { Authorization: `Bearer ${token}` }, + body: form, + }); + return { status: response.status, body: await response.json().catch(() => ({})) }; + }, { token, uploadName, uploadBody }); + if (uploadResult.status !== 200 || uploadResult.body.path !== `uploads/${uploadName}`) { + throw new Error(`User file upload failed: ${JSON.stringify(uploadResult)}`); + } + const apiVisibility = await page.evaluate(async ({ token }) => { + const headers = { Authorization: `Bearer ${token}` }; + const [status, root] = await Promise.all([ + fetch('/api/user-files/status', { headers }).then((response) => response.json()), + fetch('/api/user-files/browse', { headers }).then((response) => response.json()), + ]); + return JSON.stringify({ status, root }); + }, { token }); + for (const forbidden of ['access_key', 'secret_key', 'bucket', 'namespace', 'object_prefix']) { + if (apiVisibility.includes(forbidden)) { + throw new Error(`User file API leaked storage field: ${forbidden}`); + } + } + + const minioVerified = verifyMinio ? verifyUploadedObjectInMinio(username, uploadName) : false; + + const finalUrl = page.url(); + await page.screenshot({ path: process.env.BEAVER_SMOKE_SCREENSHOT || '/tmp/beaver-auth-files-smoke.png', fullPage: true }); + console.log(JSON.stringify({ + ok: true, + username, + email, + finalUrl, + roots: ['uploads', 'outputs', 'shared', 'tasks'], + uploadedPath: `uploads/${uploadName}`, + minioVerified, + failedResponses, + failedRequests, + }, null, 2)); + } catch (error) { + await page.screenshot({ path: process.env.BEAVER_SMOKE_ERROR_SCREENSHOT || '/tmp/beaver-auth-files-smoke-error.png', fullPage: true }).catch(() => {}); + console.error(JSON.stringify({ + ok: false, + username, + email, + currentUrl: page.url(), + error: String(error), + failedResponses, + failedRequests, + }, null, 2)); + process.exitCode = 1; + } finally { + await browser.close(); + } +} + +function verifyUploadedObjectInMinio(username, uploadName) { + const endpoint = process.env.BEAVER_SMOKE_MINIO_ENDPOINT || 'http://127.0.0.1:9000'; + const accessKey = process.env.BEAVER_SMOKE_MINIO_ACCESS_KEY || process.env.BEAVER_MINIO_ROOT_USER || ''; + const secretKey = process.env.BEAVER_SMOKE_MINIO_SECRET_KEY || process.env.BEAVER_MINIO_ROOT_PASSWORD || ''; + const bucket = process.env.BEAVER_SMOKE_MINIO_BUCKET || process.env.BEAVER_USER_FILES_BUCKET || 'beaver-user-files'; + const network = process.env.BEAVER_SMOKE_MINIO_NETWORK || ''; + if (!accessKey || !secretKey) { + throw new Error('MinIO verification requested but access key or secret key is missing'); + } + const containerArgs = ['run', '--rm', '--entrypoint', '/bin/sh']; + if (network) { + containerArgs.push('--network', network); + } + containerArgs.push( + 'minio/mc:latest', + '-lc', + [ + `mc alias set beaver ${shellQuote(endpoint)} ${shellQuote(accessKey)} ${shellQuote(secretKey)} >/dev/null`, + `mc stat ${shellQuote(`beaver/${bucket}/users/${username}/uploads/${uploadName}`)} >/dev/null`, + ].join(' && '), + ); + execFileSync('docker', containerArgs, { stdio: 'pipe' }); + return true; +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +await main(); diff --git a/scripts/smoke-auth-files.sh b/scripts/smoke-auth-files.sh new file mode 100755 index 0000000..d586548 --- /dev/null +++ b/scripts/smoke-auth-files.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +PLAYWRIGHT_WORKDIR="${BEAVER_PLAYWRIGHT_WORKDIR:-/tmp/beaver-playwright-smoke}" +PLAYWRIGHT_VERSION="${PLAYWRIGHT_VERSION:-1.60.0}" + +mkdir -p "$PLAYWRIGHT_WORKDIR" + +if ! NODE_PATH="${PLAYWRIGHT_WORKDIR}/node_modules${NODE_PATH:+:${NODE_PATH}}" node -e "require.resolve('playwright')" >/dev/null 2>&1; then + npm install --prefix "$PLAYWRIGHT_WORKDIR" --no-save "playwright@${PLAYWRIGHT_VERSION}" +fi + +export NODE_PATH="${PLAYWRIGHT_WORKDIR}/node_modules${NODE_PATH:+:${NODE_PATH}}" +exec node "${PROJECT_ROOT}/scripts/smoke-auth-files.mjs" diff --git a/scripts/validate-filesystem-automation.mjs b/scripts/validate-filesystem-automation.mjs new file mode 100755 index 0000000..5432a3b --- /dev/null +++ b/scripts/validate-filesystem-automation.mjs @@ -0,0 +1,395 @@ +#!/usr/bin/env node +import { createRequire } from 'node:module'; +import { execFileSync } from 'node:child_process'; +import { writeFileSync } from 'node:fs'; + +const require = createRequire(import.meta.url); +const { chromium } = require('playwright'); + +const now = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); +const username = process.env.BEAVER_VALIDATE_USERNAME || `fsauto${now}`; +const password = process.env.BEAVER_VALIDATE_PASSWORD || 'TestPass123!'; +const email = process.env.BEAVER_VALIDATE_EMAIL || `${username}@example.test`; +const portalUrl = (process.env.BEAVER_VALIDATE_PORTAL_URL || 'http://127.0.0.1:3081').replace(/\/$/, ''); +const executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined; +const hostResolverRules = process.env.BEAVER_VALIDATE_HOST_RESOLVER_RULES || process.env.BEAVER_SMOKE_HOST_RESOLVER_RULES || ''; +const headless = process.env.BEAVER_VALIDATE_HEADLESS !== '0'; +const verifyMinio = process.env.BEAVER_VALIDATE_VERIFY_MINIO === '1'; +const cleanupEnabled = process.env.BEAVER_VALIDATE_CLEANUP !== '0'; +const screenshotDir = process.env.BEAVER_VALIDATE_SCREENSHOT_DIR || '/tmp'; +const reportPath = process.env.BEAVER_VALIDATE_REPORT || ''; + +const taskScope = `ui-delete-${Date.now()}`; +const deleteTargets = [ + { root: 'uploads', path: 'uploads/delete-me-uploads.txt', dir: 'uploads', name: 'delete-me-uploads.txt' }, + { root: 'outputs', path: 'outputs/delete-me-outputs.txt', dir: 'outputs', name: 'delete-me-outputs.txt' }, + { root: 'shared', path: 'shared/delete-me-shared.txt', dir: 'shared', name: 'delete-me-shared.txt' }, + { root: 'tasks', path: `tasks/${taskScope}/delete-me-tasks.txt`, dir: `tasks/${taskScope}`, name: 'delete-me-tasks.txt' }, +]; + +const pageChecks = [ + { name: 'Files', path: '/files', marker: /文件管理|Files/ }, + { name: 'Tasks', path: '/tasks', marker: /任务|Tasks/ }, + { name: 'Tools/MCP', path: '/mcp', marker: /工具|Tools/ }, + { name: 'Logs', path: '/logs', marker: /运行日志|Runtime Logs/ }, + { name: 'Agents', path: '/agents', marker: /智能体|Agents/ }, + { name: 'Skills', path: '/skills', marker: /技能|Skills/ }, +]; + +const launchArgs = []; +if (hostResolverRules) { + launchArgs.push(`--host-resolver-rules=${hostResolverRules}`); +} + +const failedResponses = []; +const failedRequests = []; +const consoleErrors = []; +const pageEvidence = []; + +function remember(list, value) { + list.push(value); + if (list.length > 80) { + list.shift(); + } +} + +function sanitize(value) { + return String(value) + .replace(/(Authorization:\s*Bearer\s+)[^\s"']+/gi, '$1') + .replace(/(access[_-]?key["']?\s*[:=]\s*["']?)[^"',\s]+/gi, '$1') + .replace(/(secret[_-]?key["']?\s*[:=]\s*["']?)[^"',\s]+/gi, '$1') + .replace(/(password["']?\s*[:=]\s*["']?)[^"',\s]+/gi, '$1'); +} + +async function clickFirstVisible(locators) { + for (const locator of locators) { + if (await locator.first().isVisible().catch(() => false)) { + await locator.first().click(); + return true; + } + } + return false; +} + +async function registerAndHandoff(page) { + const registerUrl = new URL('/register', portalUrl); + registerUrl.searchParams.set('next', '/files'); + await page.goto(registerUrl.toString(), { waitUntil: 'domcontentloaded', timeout: 30000 }); + + await page.locator('#username').fill(username); + await page.locator('#email').fill(email); + await page.locator('#password').fill(password); + await page.locator('#confirmPassword').fill(password); + await page.locator('form').first().locator('button[type="submit"]').click(); + await page.waitForResponse( + (response) => response.url().includes('/api/runtime/register'), + { timeout: 180000 }, + ); + + await page.getByText(/配置模型|Model Setup/).waitFor({ timeout: 180000 }); + const skipped = await clickFirstVisible([ + page.getByRole('button', { name: /跳过|Skip/i }), + page.locator('button.secondary-button'), + ]); + if (!skipped) { + throw new Error('Model setup skip button was not found'); + } + + await page.waitForURL(/\/files$/, { timeout: 90000 }); + await page.waitForLoadState('domcontentloaded', { timeout: 30000 }).catch(() => {}); + await page.getByText(/文件管理|Files/).waitFor({ timeout: 30000 }); + + const token = await page.evaluate(() => window.localStorage.getItem('beaver_access_token') || ''); + if (!token) { + throw new Error('No Beaver access token found after app handoff'); + } + const me = await apiFetch(page, token, '/api/auth/me'); + if (me.status !== 200 || me.body?.username !== username) { + throw new Error(`App auth confirmation failed: ${JSON.stringify(me)}`); + } + assertAppHost(page.url()); + return token; +} + +function assertAppHost(url) { + const current = new URL(url); + const portal = new URL(portalUrl); + if (current.host === portal.host || current.pathname.startsWith('/login') || current.pathname.startsWith('/register')) { + throw new Error(`Browser is not on the app instance after handoff: ${url}`); + } +} + +async function apiFetch(page, token, path, options = {}) { + return page.evaluate(async ({ token, path, options }) => { + const response = await fetch(path, { + ...options, + headers: { + ...(options.headers || {}), + Authorization: `Bearer ${token}`, + }, + }); + const text = await response.text(); + let body = null; + try { + body = text ? JSON.parse(text) : null; + } catch { + body = text; + } + return { status: response.status, body }; + }, { token, path, options }); +} + +async function createDeleteTargets(page, token) { + const result = await page.evaluate(async ({ token, deleteTargets, taskScope }) => { + const headers = { Authorization: `Bearer ${token}` }; + const mkdir = await fetch(`/api/user-files/mkdir?path=${encodeURIComponent(`tasks/${taskScope}`)}`, { + method: 'POST', + headers, + }); + if (!mkdir.ok && mkdir.status !== 409) { + throw new Error(`mkdir failed: ${mkdir.status}`); + } + const uploaded = []; + for (const target of deleteTargets) { + const form = new FormData(); + form.set('path', target.dir); + form.set('file', new File([`delete target ${target.path}`], target.name, { type: 'text/plain' })); + const response = await fetch('/api/user-files/upload', { + method: 'POST', + headers, + body: form, + }); + const body = await response.json().catch(() => ({})); + if (!response.ok || body.path !== target.path) { + throw new Error(`upload failed for ${target.path}: ${response.status} ${JSON.stringify(body)}`); + } + uploaded.push(body.path); + } + return uploaded; + }, { token, deleteTargets, taskScope }); + if (result.length !== deleteTargets.length) { + throw new Error(`Unexpected upload target count: ${result.length}`); + } +} + +async function waitForRootEntries(page) { + await page.getByText(/文件管理|Files/).waitFor({ timeout: 30000 }); + for (const root of ['uploads', 'outputs', 'shared', 'tasks']) { + await page.locator('button').filter({ hasText: root }).first().waitFor({ state: 'visible', timeout: 30000 }); + } +} + +async function navigateToRoot(page) { + await page.goto(`${new URL(page.url()).origin}/files`, { waitUntil: 'domcontentloaded', timeout: 30000 }); + await waitForRootEntries(page); +} + +async function openDirectory(page, name) { + const row = page.locator('button').filter({ hasText: name }).first(); + await row.waitFor({ state: 'visible', timeout: 30000 }); + await row.click(); + await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {}); +} + +async function deleteFromFilesPage(page, target) { + await navigateToRoot(page); + await openDirectory(page, target.root); + if (target.root === 'tasks') { + await openDirectory(page, taskScope); + } + + const row = page.locator('button').filter({ hasText: target.name }).first(); + await row.waitFor({ state: 'visible', timeout: 30000 }); + await row.hover(); + await row.locator('[title="Delete"], [title="删除"]').last().click({ force: true }); + await page.waitForFunction( + (name) => !document.body.innerText.includes(name), + target.name, + { timeout: 30000 }, + ); +} + +async function verifyDeletedViaApi(page, token) { + for (const target of deleteTargets) { + const preview = await apiFetch(page, token, `/api/user-files/preview?path=${encodeURIComponent(target.path)}`); + if (preview.status !== 404) { + throw new Error(`Expected preview 404 after UI delete for ${target.path}, got ${preview.status}`); + } + } + const roots = await apiFetch(page, token, '/api/user-files/browse'); + if (roots.status !== 200) { + throw new Error(`Root browse failed after UI deletion: ${roots.status}`); + } + const names = new Set((roots.body?.items || []).map((item) => item.name)); + for (const root of ['uploads', 'outputs', 'shared', 'tasks']) { + if (!names.has(root)) { + throw new Error(`Virtual root disappeared after UI deletion: ${root}`); + } + } +} + +function verifyDeletedInMinio() { + if (!verifyMinio) { + return false; + } + const endpoint = process.env.BEAVER_VALIDATE_MINIO_ENDPOINT || process.env.BEAVER_SMOKE_MINIO_ENDPOINT || 'http://127.0.0.1:9000'; + const accessKey = process.env.BEAVER_VALIDATE_MINIO_ACCESS_KEY || process.env.BEAVER_SMOKE_MINIO_ACCESS_KEY || process.env.BEAVER_MINIO_ROOT_USER || ''; + const secretKey = process.env.BEAVER_VALIDATE_MINIO_SECRET_KEY || process.env.BEAVER_SMOKE_MINIO_SECRET_KEY || process.env.BEAVER_MINIO_ROOT_PASSWORD || ''; + const bucket = process.env.BEAVER_VALIDATE_MINIO_BUCKET || process.env.BEAVER_SMOKE_MINIO_BUCKET || process.env.BEAVER_USER_FILES_BUCKET || 'beaver-user-files'; + const network = process.env.BEAVER_VALIDATE_MINIO_NETWORK || process.env.BEAVER_SMOKE_MINIO_NETWORK || ''; + if (!accessKey || !secretKey) { + throw new Error('MinIO verification requested but access key or secret key is missing'); + } + const checks = deleteTargets + .map((target) => `if mc stat ${shellQuote(`beaver/${bucket}/users/${username}/${target.path}`)} >/dev/null 2>&1; then echo ${shellQuote(`object still exists: ${target.path}`)}; exit 1; fi`) + .join(' && '); + const containerArgs = ['run', '--rm', '--entrypoint', '/bin/sh']; + if (network) { + containerArgs.push('--network', network); + } + containerArgs.push( + 'minio/mc:latest', + '-lc', + [ + `mc alias set beaver ${shellQuote(endpoint)} ${shellQuote(accessKey)} ${shellQuote(secretKey)} >/dev/null`, + checks, + ].join(' && '), + ); + execFileSync('docker', containerArgs, { stdio: 'pipe' }); + return true; +} + +async function checkPage(page, appOrigin, check) { + const beforeResponseCount = failedResponses.length; + const beforeConsoleCount = consoleErrors.length; + await page.goto(`${appOrigin}${check.path}`, { waitUntil: 'domcontentloaded', timeout: 30000 }); + await page.getByText(check.marker).first().waitFor({ timeout: 30000 }); + if (check.path === '/agents') { + const subagentTab = page.getByRole('tab', { name: /Persistent sub-agents|Persistent Sub-Agents|Sub-Agent/i }).first(); + if (await subagentTab.isVisible().catch(() => false)) { + await subagentTab.click(); + await page.getByText(/Sub-Agent|sub-agent|子智能体/i).first().waitFor({ timeout: 30000 }); + } + } + await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {}); + assertAppHost(page.url()); + const newResponses = failedResponses.slice(beforeResponseCount); + const newConsoleErrors = consoleErrors.slice(beforeConsoleCount); + const hardFailures = newResponses.filter((entry) => entry.status >= 500 || (entry.url.includes('/api/user-files/') && entry.status >= 400)); + if (hardFailures.length) { + throw new Error(`Page ${check.name} had failing responses: ${hardFailures.map((item) => `${item.status} ${item.url}`).join('; ')}`); + } + const fileConsoleErrors = newConsoleErrors.filter((entry) => /user[-_ ]?files|filesystem|minio|storage/i.test(entry.text)); + if (fileConsoleErrors.length) { + throw new Error(`Page ${check.name} had file-system-related console errors: ${fileConsoleErrors.map((item) => item.text).join('; ')}`); + } + pageEvidence.push({ name: check.name, path: check.path, finalUrl: page.url(), failedResponseCount: newResponses.length }); +} + +async function cleanupDisposableUser() { + const token = process.env.BEAVER_VALIDATE_DEPLOY_TOKEN || process.env.DEPLOY_CONTROL_API_TOKEN || ''; + const deployUrl = (process.env.BEAVER_VALIDATE_DEPLOY_CONTROL_URL || process.env.DEPLOY_CONTROL_URL || 'http://127.0.0.1:8090').replace(/\/$/, ''); + if (!cleanupEnabled || !token) { + return { attempted: false, reason: cleanupEnabled ? 'missing deploy-control token' : 'disabled' }; + } + const response = await fetch(`${deployUrl}/api/instances/${encodeURIComponent(username)}`, { + method: 'DELETE', + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/json', + 'X-Purge-Data': '1', + 'X-Purge-User-Files': '1', + }, + }); + const body = await response.json().catch(() => ({})); + return { attempted: true, status: response.status, ok: response.ok, body }; +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +async function main() { + const browser = await chromium.launch({ headless, executablePath, args: launchArgs }); + const page = await browser.newPage({ viewport: { width: 1365, height: 900 } }); + page.on('dialog', (dialog) => dialog.accept()); + page.on('response', (response) => { + if (response.status() >= 400) { + remember(failedResponses, { status: response.status(), url: response.url() }); + } + }); + page.on('requestfailed', (request) => { + remember(failedRequests, { url: request.url(), error: request.failure()?.errorText || '' }); + }); + page.on('console', (message) => { + if (message.type() === 'error') { + remember(consoleErrors, { text: sanitize(message.text()) }); + } + }); + + let cleanup = { attempted: false, reason: 'not reached' }; + try { + const token = await registerAndHandoff(page); + const appOrigin = new URL(page.url()).origin; + await waitForRootEntries(page); + await createDeleteTargets(page, token); + + for (const target of deleteTargets) { + await deleteFromFilesPage(page, target); + } + + await verifyDeletedViaApi(page, token); + const minioVerified = verifyDeletedInMinio(); + + for (const check of pageChecks) { + await checkPage(page, appOrigin, check); + } + + const finalScreenshot = `${screenshotDir}/beaver-filesystem-validation-${username}.png`; + await page.screenshot({ path: finalScreenshot, fullPage: true }).catch(() => {}); + cleanup = await cleanupDisposableUser(); + const result = { + ok: true, + username, + email, + deletedRoots: deleteTargets.map((target) => target.root), + deletedPaths: deleteTargets.map((target) => target.path), + checkedPages: pageEvidence, + minioVerified, + cleanup, + screenshot: finalScreenshot, + failedResponses, + failedRequests, + consoleErrors, + }; + if (reportPath) { + writeFileSync(reportPath, `${JSON.stringify(result, null, 2)}\n`, 'utf8'); + } + console.log(JSON.stringify(result, null, 2)); + } catch (error) { + const errorScreenshot = `${screenshotDir}/beaver-filesystem-validation-${username}-error.png`; + await page.screenshot({ path: errorScreenshot, fullPage: true }).catch(() => {}); + cleanup = await cleanupDisposableUser().catch((cleanupError) => ({ attempted: true, ok: false, error: sanitize(cleanupError) })); + const result = { + ok: false, + username, + email, + currentUrl: page.url(), + error: sanitize(error), + cleanup, + screenshot: errorScreenshot, + failedResponses, + failedRequests, + consoleErrors, + }; + if (reportPath) { + writeFileSync(reportPath, `${JSON.stringify(result, null, 2)}\n`, 'utf8'); + } + console.error(JSON.stringify(result, null, 2)); + process.exitCode = 1; + } finally { + await browser.close(); + } +} + +await main(); diff --git a/scripts/validate-filesystem-automation.sh b/scripts/validate-filesystem-automation.sh new file mode 100755 index 0000000..eb3b5aa --- /dev/null +++ b/scripts/validate-filesystem-automation.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +PLAYWRIGHT_WORKDIR="${BEAVER_PLAYWRIGHT_WORKDIR:-/tmp/beaver-playwright-smoke}" +PLAYWRIGHT_VERSION="${PLAYWRIGHT_VERSION:-1.60.0}" + +mkdir -p "$PLAYWRIGHT_WORKDIR" + +if ! NODE_PATH="${PLAYWRIGHT_WORKDIR}/node_modules${NODE_PATH:+:${NODE_PATH}}" node -e "require.resolve('playwright')" >/dev/null 2>&1; then + npm install --prefix "$PLAYWRIGHT_WORKDIR" --no-save "playwright@${PLAYWRIGHT_VERSION}" +fi + +export NODE_PATH="${PLAYWRIGHT_WORKDIR}/node_modules${NODE_PATH:+:${NODE_PATH}}" +exec node "${PROJECT_ROOT}/scripts/validate-filesystem-automation.mjs" diff --git a/域名配置指引.md b/域名配置指引.md index 68a12b5..f8bcf8e 100644 --- a/域名配置指引.md +++ b/域名配置指引.md @@ -1,6 +1,6 @@ # Beaver Project 域名配置指引 -这份文档说明如何从本机测试域名 `127.0.0.1.nip.io` 切换到正式域名。 +这份文档说明如何从本机测试域名 `localhost` 子域名切换到正式域名。 核心结论： @@ -105,7 +105,7 @@ proxy_set_header X-Forwarded-Proto $scheme; 本机测试： ```bash -export BEAVER_BASE_DOMAIN=127.0.0.1.nip.io +export BEAVER_BASE_DOMAIN=localhost ``` `deploy-control`： @@ -119,7 +119,7 @@ export BEAVER_BASE_DOMAIN=127.0.0.1.nip.io 生成实例地址： ```text -http://alice.127.0.0.1.nip.io:8088 +http://alice.localhost:8088 ``` 正式 HTTPS： @@ -322,7 +322,7 @@ portal.example.com -> 3081 如果只是本机验证完整链路，继续使用： ```bash -export BEAVER_BASE_DOMAIN=127.0.0.1.nip.io +export BEAVER_BASE_DOMAIN=localhost ``` 它已经足够测试： diff --git a/部署指南.md b/部署指南.md index 21ccc40..a7ecee3 100644 --- a/部署指南.md +++ b/部署指南.md @@ -12,7 +12,7 @@ - 浏览器能打开 `http://127.0.0.1:3081/register` - 注册账号后自动创建专属实例 -- 浏览器跳转到 `http://.127.0.0.1.nip.io:8088` +- 浏览器跳转到 `http://.localhost:8088` 如果你只单独启动某个前端页面，页面可以打开，但注册、登录、创建实例这些动作不一定能通。 @@ -58,14 +58,21 @@ pwd ## 2. 准备本机测试变量 -本机测试推荐用 `127.0.0.1.nip.io`。例如： +本机测试推荐用 `localhost` 子域名。例如： ```text -alice.127.0.0.1.nip.io -> 127.0.0.1 +alice.localhost -> 127.0.0.1 / ::1 ``` 这样 `router-proxy` 可以按子域名区分不同实例。 +注意： + +- `*.localhost` 只适合在部署机器本机浏览器里测试。 +- 如果别的电脑访问 `alice.localhost`，它会指向那台电脑自己，不会指向 Beaver 服务器。 +- 局域网、服务器或正式环境必须把 `BEAVER_BASE_DOMAIN` 改成客户端能解析到 Beaver 服务器的域名，例如 `apps.example.com`。 +- 旧版文档使用过 `127.0.0.1.nip.io`。它本来应该按域名里的 IP 解析到 `127.0.0.1`，但在部分 VPN、代理、DNS 网关或内网安全设备环境下会被改写到 `198.18.0.0/15` 这类假 IP 段，导致浏览器连不上本机服务。 + 直接执行： ```bash @@ -76,11 +83,17 @@ export BEAVER_PROXY_CONTAINER_NAME=beaver-router-proxy export BEAVER_DEPLOY_TOKEN="$(openssl rand -hex 32)" export BEAVER_AUTHZ_INTERNAL_TOKEN="$(openssl rand -hex 32)" -export BEAVER_BASE_DOMAIN=127.0.0.1.nip.io +export BEAVER_BASE_DOMAIN=localhost export BEAVER_AUTHZ_URL='http://beaver-authz-service:19090' export BEAVER_DEPLOY_URL='http://beaver-deploy-control:8090' +export BEAVER_MINIO_ROOT_USER='beaver-minio-admin' +export BEAVER_MINIO_ROOT_PASSWORD="$(openssl rand -hex 32)" +export BEAVER_USER_FILES_BUCKET='beaver-user-files' +export BEAVER_USER_FILES_MINIO_ENDPOINT='beaver-minio:9000' +export BEAVER_USER_FILES_MAX_UPLOAD_BYTES=$((5 * 1024 * 1024 * 1024)) + export BEAVER_OUTLOOK_MCP_URL='' export BEAVER_OUTLOOK_MCP_SERVER_ID='outlook_mcp' ``` @@ -93,12 +106,32 @@ export BEAVER_OUTLOOK_MCP_SERVER_ID='outlook_mcp' | `BEAVER_NET` | 所有容器共用的 Docker network | | `BEAVER_DEPLOY_TOKEN` | `auth-portal` / `authz-service` 调 `deploy-control` 的 token | | `BEAVER_AUTHZ_INTERNAL_TOKEN` | AuthZ 内部接口 token | -| `BEAVER_BASE_DOMAIN` | 新实例的基域名 | +| `BEAVER_BASE_DOMAIN` | 新实例的基域名；本机测试用 `localhost`，服务器部署用真实域名 | | `BEAVER_AUTHZ_URL` | 容器网络内访问 AuthZ 的地址 | | `BEAVER_DEPLOY_URL` | 容器网络内访问 deploy-control 的地址 | +| `BEAVER_MINIO_ROOT_USER` / `BEAVER_MINIO_ROOT_PASSWORD` | 只给 provisioning 组件使用的 MinIO 管理凭据 | +| `BEAVER_USER_FILES_BUCKET` | 用户文件系统共用 bucket，默认 `beaver-user-files` | +| `BEAVER_USER_FILES_MINIO_ENDPOINT` | 容器网络内访问 MinIO API 的地址 | +| `BEAVER_USER_FILES_MAX_UPLOAD_BYTES` | 用户文件系统上传上限，默认 5GB；聊天附件和 workspace 上传仍保留当前小文件限制 | | `BEAVER_OUTLOOK_MCP_URL` | 可选 Outlook MCP HTTP 地址 | | `BEAVER_OUTLOOK_MCP_SERVER_ID` | Outlook MCP server id，默认 `outlook_mcp` | +如果接入外部正式 MinIO，不需要启动本地 `beaver-minio`。把上面的 MinIO 变量改成正式服务即可： + +```bash +export BEAVER_MINIO_ROOT_USER='' +export BEAVER_MINIO_ROOT_PASSWORD='' +export BEAVER_USER_FILES_BUCKET='beaver-user-files-formal-test' +export BEAVER_USER_FILES_MINIO_ENDPOINT='10.6.80.98:19000' +``` + +注意： + +- `BEAVER_USER_FILES_MINIO_ENDPOINT` 是给 Python MinIO SDK 用的 S3 API endpoint，格式是 `host:port`，不要带 `http://`。 +- 操作员用 `mc` 或 `curl` 验证时才写成 `http://10.6.80.98:19000`。 +- MinIO Console 端口不是 S3 API 端口。例如 `19001` 如果返回 MinIO Console 页面，就不能填进 `USER_FILES_MINIO_ENDPOINT`。 +- 这个管理员账号只给 AuthZ provisioning 使用，用于创建共享 bucket、scoped user 和 scoped policy；不要暴露给前端或普通用户。 + `BEAVER_AUTHZ_URL` 和 `BEAVER_DEPLOY_URL` 必须带协议头。正确写法： ```text @@ -126,11 +159,37 @@ beaver-deploy-control:8090 - `beaver-authz-service` - `beaver-auth-portal` +如果改的是 `BEAVER_BASE_DOMAIN`，还要重启 `beaver-deploy-control`。这个变量只影响之后新创建的实例；已经创建过的实例 URL 已经写入 `app-instance/runtime/registry/instances.json`，不会自动改成新域名。 + +### 非本机访问怎么配置域名 + +如果 Beaver 部署在服务器上，而用户从其他机器访问，不要使用 `localhost`。推荐准备一个真实域名，并把通配子域名解析到服务器，例如： + +```text +portal.example.com -> Beaver 服务器 +*.apps.example.com -> Beaver 服务器 +``` + +然后使用： + +```bash +export BEAVER_BASE_DOMAIN=apps.example.com +``` + +新实例 URL 会变成： + +```text +http://alice.apps.example.com:8088 +``` + +如果外层 Nginx/Caddy/负载均衡已经把 `https://*.apps.example.com` 转发到 `router-proxy`，正式环境通常还会把 `DEPLOY_PUBLIC_SCHEME` 改为 `https`，并把 `DEPLOY_PUBLIC_PORT` 改成 `443`。 + ## 3. 创建运行目录 ```bash mkdir -p \ "$PROJECT_ROOT/authz-service/runtime/data" \ + "$PROJECT_ROOT/minio/runtime/data" \ "$PROJECT_ROOT/app-instance/runtime/instances" \ "$PROJECT_ROOT/app-instance/runtime/registry" \ "$PROJECT_ROOT/router-proxy/runtime/conf.d" @@ -139,6 +198,7 @@ mkdir -p \ 这些目录保存： - AuthZ 数据 +- MinIO 对象数据 - 实例注册表 - 每个用户实例的配置和数据 - `router-proxy` 生成的路由文件 @@ -183,16 +243,57 @@ PROXY_HTTP_PORT=8088 \ 实例统一入口： ```text -http://.127.0.0.1.nip.io:8088 +http://.localhost:8088 ``` 示例： ```text -http://alice.127.0.0.1.nip.io:8088 +http://alice.localhost:8088 ``` -## 7. 启动 authz-service +这里的 `localhost` 示例只表示本机测试。服务器部署时应替换为上面配置的真实基域名，例如： + +```text +http://alice.apps.example.com:8088 +``` + +## 7. 启动 MinIO + +MinIO 是用户文件系统的后端实现细节。用户和前端不会看到 bucket、access key 或 prefix；Beaver 只通过 `/api/user-files/*` 暴露个人智能体文件系统。 + +如果使用外部正式 MinIO，可以跳过本节本地 MinIO 容器启动，直接进入 `authz-service` 启动步骤。 + +```bash +docker rm -f beaver-minio >/dev/null 2>&1 || true + +docker run -d \ + --name beaver-minio \ + --restart unless-stopped \ + --network "$BEAVER_NET" \ + -p 9000:9000 \ + -p 9001:9001 \ + -v "$PROJECT_ROOT/minio/runtime/data:/data" \ + -e MINIO_ROOT_USER="$BEAVER_MINIO_ROOT_USER" \ + -e MINIO_ROOT_PASSWORD="$BEAVER_MINIO_ROOT_PASSWORD" \ + minio/minio:latest server /data --console-address ":9001" +``` + +用户文件采用共享 bucket + 用户 namespace： + +```text +bucket: beaver-user-files +namespace: users/{backend_id} +example object: users/alice/uploads/report.pdf +``` + +每个 backend/user 会由 AuthZ provisioning 生成 scoped MinIO 凭据，policy 只允许访问自己的 `users/{backend_id}/*` prefix。 + +用户文件路径必须使用相对的虚拟根路径，例如 `uploads/input.txt`、`outputs/report.md`、`shared/profile.json`、`tasks//draft.md`。`/uploads/input.txt` 这类 leading-slash absolute-style path 会被拒绝，不会再被规范化成当前用户的 `uploads/input.txt`。 + +用户文件上传由 Beaver 后端代理到 MinIO，不暴露 bucket、prefix 或凭据。当前默认允许最大 5GB 的用户文件上传，业务上限由 app-instance 后端环境变量 `BEAVER_USER_FILES_MAX_UPLOAD_BYTES` 控制；反向代理默认 `client_max_body_size` 已提高到 5GB。MinIO 本身支持大对象和 multipart 上传，但 agent 对超大文件的读取/处理能力仍需要按具体任务另行验证。 + +## 8. 启动 authz-service ```bash docker rm -f beaver-authz-service >/dev/null 2>&1 || true @@ -207,6 +308,13 @@ docker run -d \ -e AUTHZ_INTERNAL_TOKEN="$BEAVER_AUTHZ_INTERNAL_TOKEN" \ -e DEPLOY_API_BASE_URL="$BEAVER_DEPLOY_URL" \ -e DEPLOY_API_TOKEN="$BEAVER_DEPLOY_TOKEN" \ + -e USER_FILES_MINIO_PROVISIONING_ENABLED=1 \ + -e USER_FILES_MINIO_ENDPOINT="$BEAVER_USER_FILES_MINIO_ENDPOINT" \ + -e USER_FILES_MINIO_PUBLIC_ENDPOINT="$BEAVER_USER_FILES_MINIO_ENDPOINT" \ + -e USER_FILES_MINIO_ADMIN_ACCESS_KEY="$BEAVER_MINIO_ROOT_USER" \ + -e USER_FILES_MINIO_ADMIN_SECRET_KEY="$BEAVER_MINIO_ROOT_PASSWORD" \ + -e USER_FILES_MINIO_BUCKET="$BEAVER_USER_FILES_BUCKET" \ + -e USER_FILES_MINIO_SECURE=0 \ beaver/authz-service:latest ``` @@ -215,15 +323,16 @@ docker run -d \ - `AUTHZ_ISSUER` 在当前部署里要写 `http://beaver-authz-service:19090` - 不要写 `http://127.0.0.1:19090` - 新创建的 `app-instance` 容器要通过 Docker network 访问 AuthZ +- `USER_FILES_MINIO_*` 只用于 AuthZ provisioning 创建 bucket、用户、policy，并把 scoped settings 存入 AuthZ；普通用户不会接触这些配置。 检查关键环境变量： ```bash docker inspect beaver-authz-service --format '{{range .Config.Env}}{{println .}}{{end}}' \ - | egrep '^(AUTHZ_ISSUER|DEPLOY_API_BASE_URL)=' + | egrep '^(AUTHZ_ISSUER|DEPLOY_API_BASE_URL|USER_FILES_MINIO_)=' ``` -## 8. 启动 deploy-control +## 9. 启动 deploy-control `deploy-control` 会挂载 Docker socket，再创建新的 `app-instance` 容器。这里最容易错的是路径挂载： @@ -252,8 +361,10 @@ docker run -d \ -e APP_INSTANCE_IMAGE="beaver/app-instance:latest" \ -e APP_INSTANCE_NETWORK_NAME="$BEAVER_NET" \ -e DEFAULT_AUTHZ_BASE_URL="$BEAVER_AUTHZ_URL" \ + -e DEFAULT_AUTHZ_INTERNAL_TOKEN="$BEAVER_AUTHZ_INTERNAL_TOKEN" \ -e DEFAULT_AUTHZ_OUTLOOK_MCP_URL="$BEAVER_OUTLOOK_MCP_URL" \ -e DEFAULT_OUTLOOK_MCP_SERVER_ID="$BEAVER_OUTLOOK_MCP_SERVER_ID" \ + -e DEFAULT_USER_FILES_MAX_UPLOAD_BYTES="$BEAVER_USER_FILES_MAX_UPLOAD_BYTES" \ -e DEPLOY_PUBLIC_SCHEME="http" \ -e DEPLOY_PUBLIC_BASE_DOMAIN="$BEAVER_BASE_DOMAIN" \ -e DEPLOY_PUBLIC_PORT="8088" \ @@ -261,9 +372,13 @@ docker run -d \ beaver/deploy-control:latest ``` +`DEPLOY_PUBLIC_BASE_DOMAIN` 来自 `BEAVER_BASE_DOMAIN`。本机测试时可以是 `localhost`；如果要让其他设备访问，必须换成它们能解析到 Beaver 服务器的真实域名。修改后需要重启 `beaver-deploy-control`，并重新创建实例或手动更新 registry 后重载 `router-proxy`。 + 当前版本创建实例时会传 `--skip-provider-config`，也就是先不写 provider、model 或 API key。注册成功后，`auth-portal` 会进入模型配置引导页，再调用 `deploy-control /api/instances/configure-provider` 写入该实例的 `config.json` 并重启容器。 -## 9. 启动 auth-portal +`DEFAULT_AUTHZ_INTERNAL_TOKEN` 会写入新建 app-instance 的后端 runtime env，用于 app-instance 后端读取自己的 internal MinIO settings。它不会传给前端。 + +## 10. 启动 auth-portal ```bash docker rm -f beaver-auth-portal >/dev/null 2>&1 || true @@ -286,12 +401,13 @@ docker inspect beaver-auth-portal --format '{{range .Config.Env}}{{println .}}{{ | egrep '^(AUTHZ_API_BASE_URL|DEPLOY_API_BASE_URL)=' ``` -## 10. 健康检查 +## 11. 健康检查 ```bash curl http://127.0.0.1:19090/healthz curl http://127.0.0.1:8090/healthz curl -I http://127.0.0.1:3081 +curl -I http://127.0.0.1:9001 docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' docker logs --tail=50 beaver-router-proxy ``` @@ -299,11 +415,12 @@ docker logs --tail=50 beaver-router-proxy 至少应该看到这些容器： - `beaver-authz-service` +- `beaver-minio` - `beaver-deploy-control` - `beaver-auth-portal` - `beaver-router-proxy` -## 11. 浏览器首次测试 +## 12. 浏览器首次测试 打开： @@ -322,10 +439,140 @@ http://127.0.0.1:3081/register 跳转目标示例： ```text -http://alice.127.0.0.1.nip.io:8088 +http://alice.localhost:8088 ``` -## 12. 确认实例已创建 +也可以运行 Playwright 冒烟测试，自动验证注册、跳过模型配置、登录交接、文件页四个根目录、上传文件，并检查 `/api/user-files/*` 不泄露 bucket、namespace 或凭据字段： + +```bash +cd "$PROJECT_ROOT" +PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/home/worker/.cache/ms-playwright/chromium-1194/chrome-linux/chrome \ + ./scripts/smoke-auth-files.sh +``` + +如果要同时验证上传后的对象确实进入 MinIO 的 `users/{backend_id}/uploads/` namespace，可以加上 MinIO 校验： + +```bash +BEAVER_SMOKE_VERIFY_MINIO=1 \ +BEAVER_SMOKE_MINIO_ENDPOINT=http://beaver-minio:9000 \ +BEAVER_SMOKE_MINIO_ACCESS_KEY="$BEAVER_MINIO_ROOT_USER" \ +BEAVER_SMOKE_MINIO_SECRET_KEY="$BEAVER_MINIO_ROOT_PASSWORD" \ +BEAVER_SMOKE_MINIO_BUCKET="$BEAVER_USER_FILES_BUCKET" \ +BEAVER_SMOKE_MINIO_NETWORK="$BEAVER_NET" \ +PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/home/worker/.cache/ms-playwright/chromium-1194/chrome-linux/chrome \ + ./scripts/smoke-auth-files.sh +``` + +如果你的机器没有这个 Chromium 路径，可以去掉 `PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH`，让 Playwright 使用自己安装的浏览器。 + +外部正式 MinIO 的重复验证流程： + +```bash +cd "$PROJECT_ROOT" + +BEAVER_SMOKE_VERIFY_MINIO=1 \ +BEAVER_SMOKE_MINIO_ENDPOINT=http://10.6.80.98:19000 \ +BEAVER_SMOKE_MINIO_ACCESS_KEY="$BEAVER_MINIO_ROOT_USER" \ +BEAVER_SMOKE_MINIO_SECRET_KEY="$BEAVER_MINIO_ROOT_PASSWORD" \ +BEAVER_SMOKE_MINIO_BUCKET="$BEAVER_USER_FILES_BUCKET" \ +PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/home/worker/.cache/ms-playwright/chromium-1194/chrome-linux/chrome \ + ./scripts/smoke-auth-files.sh +``` + +预期结果： + +- 新注册用户的 AuthZ MinIO settings 指向正式 endpoint 和 `users/{backend_id}` namespace。 +- 文件页只展示 `uploads`、`outputs`、`shared`、`tasks` 四个根目录。 +- `/api/user-files/*` 上传的对象出现在正式 bucket 的 `users/{backend_id}/uploads/` 下。 +- 覆盖、下载、删除都由 Beaver API 完成，前端响应不包含 bucket、namespace、access key 或 secret key。 + +也可以运行更完整的用户文件系统验证自动化，覆盖 Files 页面四根目录 UI 删除、Logs/Subagents 等页面回归、MinIO 缺失对象验证和临时用户清理： + +```bash +cd "$PROJECT_ROOT" + +BEAVER_VALIDATE_VERIFY_MINIO=1 \ +BEAVER_VALIDATE_MINIO_ENDPOINT=http://10.6.80.98:19000 \ +BEAVER_VALIDATE_MINIO_ACCESS_KEY="$BEAVER_MINIO_ROOT_USER" \ +BEAVER_VALIDATE_MINIO_SECRET_KEY="$BEAVER_MINIO_ROOT_PASSWORD" \ +BEAVER_VALIDATE_MINIO_BUCKET="$BEAVER_USER_FILES_BUCKET" \ +BEAVER_VALIDATE_DEPLOY_TOKEN="$BEAVER_DEPLOY_TOKEN" \ +PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/home/worker/.cache/ms-playwright/chromium-1194/chrome-linux/chrome \ + ./scripts/validate-filesystem-automation.sh +``` + +### 清理测试账号和 MinIO 用户文件 + +测试注册、Playwright smoke 或反复部署会创建 app-instance、本地实例目录、AuthZ MinIO settings，以及 MinIO 里的 scoped user、policy 和 `users/{backend_id}/` 对象。不要只手工删除 local path，否则 MinIO 里会留下无效测试资源。 + +优先使用清理脚本。默认是 dry-run，只列出将要清理的测试账号： + +```bash +cd "$PROJECT_ROOT" +./scripts/cleanup-test-users.py --username-prefix smoke +``` + +确认列表无误后执行删除： + +```bash +DEPLOY_CONTROL_API_TOKEN="$BEAVER_DEPLOY_TOKEN" \ +./scripts/cleanup-test-users.py \ + --username-prefix smoke \ + --purge-data \ + --execute +``` + +这会调用 `deploy-control` 的实例删除接口，并带上： + +```text +X-Purge-Data: 1 +X-Purge-User-Files: 1 +``` + +其中 `X-Purge-Data` 删除本地实例数据，`X-Purge-User-Files` 让 deploy-control 调 AuthZ 内部接口清理 MinIO 用户文件资源。用户和普通 Files 页面不会看到 bucket、access key、secret key、policy 或 raw prefix；MinIO 仍然只是后端实现细节。 + +如果同一个实例删除请求重复执行，deploy-control 会把本地实例已不存在报告为 `already_absent` 的成功 no-op。带 `X-Purge-User-Files: 1` 时，它仍会对同名 backend id 调用 AuthZ 的 best-effort 用户文件清理；AuthZ user-file cleanup 本身是幂等的，已不存在的 scoped user、policy、settings 或 namespace 会以 absent/no-op 状态返回。 + +如果只想清理一个明确实例，也可以直接调用： + +```bash +curl -X DELETE "http://127.0.0.1:8090/api/instances/" \ + -H "Authorization: Bearer $BEAVER_DEPLOY_TOKEN" \ + -H "X-Purge-Data: 1" \ + -H "X-Purge-User-Files: 1" +``` + +如果删除过程中只有一部分成功，可以按下面的方式手工恢复。先设置 MinIO alias： + +```bash +docker run --rm --network "$BEAVER_NET" --entrypoint /bin/sh minio/mc:latest -lc " + mc alias set beaver http://beaver-minio:9000 '$BEAVER_MINIO_ROOT_USER' '$BEAVER_MINIO_ROOT_PASSWORD' +" +``` + +然后针对某个 backend id 清理对象、policy 和 user： + +```bash +BACKEND_ID='' +ACCESS_KEY="beaver-$BACKEND_ID" +POLICY_NAME="beaver-user-files-$BACKEND_ID" + +docker run --rm --network "$BEAVER_NET" --entrypoint /bin/sh minio/mc:latest -lc " + mc alias set beaver http://beaver-minio:9000 '$BEAVER_MINIO_ROOT_USER' '$BEAVER_MINIO_ROOT_PASSWORD' >/dev/null && + mc rm --recursive --force 'beaver/$BEAVER_USER_FILES_BUCKET/users/$BACKEND_ID/' || true && + mc admin policy detach beaver '$POLICY_NAME' --user '$ACCESS_KEY' || true && + mc admin user remove beaver '$ACCESS_KEY' || true && + mc admin policy remove beaver '$POLICY_NAME' || true +" +``` + +最后删除 AuthZ 里的 MinIO settings： + +```bash +curl -X DELETE "http://127.0.0.1:19090/backends/$BACKEND_ID/settings/minio" +``` + +## 13. 确认实例已创建 ```bash cd "$PROJECT_ROOT/app-instance" @@ -343,7 +590,7 @@ docker ps --format 'table {{.Names}}\t{{.Status}}' | grep app-instance - `public_url` - `instance_host` -## 13. 只看 auth-portal 页面 +## 14. 只看 auth-portal 页面 如果只想看 Portal 页面，不跑全链路： @@ -361,7 +608,7 @@ http://127.0.0.1:3081 注意：这只能看页面。注册、登录、创建实例仍依赖 `authz-service` 和 `deploy-control`。 -## 14. 常用排错命令 +## 15. 常用排错命令 ```bash docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' @@ -396,7 +643,7 @@ docker inspect beaver-auth-portal --format '{{range .Config.Env}}{{println .}}{{ 它们都必须是完整 URL，不能是空字符串，也不能是裸 `host:port`。 -## 15. 常见问题 +## 16. 常见问题 ### 注册页报 URL 缺少协议 @@ -447,15 +694,30 @@ $PROJECT_ROOT/router-proxy -> $PROJECT_ROOT/router-proxy 因为 `deploy-control` 会通过宿主机 Docker socket 再创建新容器，传给 Docker 的 bind mount 源路径必须是宿主机真实路径。 -### `nip.io` 解析失败 +### 本地域名解析失败 检查： ```bash -ping 127.0.0.1.nip.io +getent hosts alice.localhost ``` -如果本地网络屏蔽了 `nip.io`，子域名测试会失败。可以临时换成本机 hosts 或正式域名。 +如果浏览器或系统没有把 `.localhost` 解析到本机，可以临时在 `/etc/hosts` 添加当前测试账号的主机名： + +```text +127.0.0.1 alice.localhost +``` + +旧版文档使用过 `127.0.0.1.nip.io`，但部分网络会把 `nip.io` / `sslip.io` / `lvh.me` 劫持到非本机地址。例如 `127.0.0.1.nip.io` 可能被解析成 `198.18.1.27`，这通常是代理、VPN 或 DNS 网关返回的假 IP，不是 Beaver 服务地址。遇到这种情况，本机测试优先使用 `localhost` 子域名。 + +### 服务器上不能用 `.localhost` + +`localhost` 是保留域名，浏览器会把它解析到“当前这台客户端机器”。所以： + +- 在 Beaver 服务器本机浏览器里打开 `alice.localhost`，访问的是 Beaver 服务器本机。 +- 在另一台电脑浏览器里打开 `alice.localhost`，访问的是那台电脑自己。 + +因此远程访问、局域网访问和正式部署都不能使用 `*.localhost`。请改用真实域名或可被客户端解析到服务器的内部域名，并保证 `router-proxy` 能收到原始 `Host` 头。 ### 端口被占用 @@ -472,7 +734,7 @@ ping 127.0.0.1.nip.io ss -ltnp | grep -E '3081|8088|8090|19090' ``` -## 16. 重新部署基础容器 +## 17. 重新部署基础容器 只重建基础四个容器：