feat: integrate MinIO-backed user filesystem
This commit is contained in:
Codex
@ -18,7 +18,7 @@ DEFAULT_AUTHZ_OUTLOOK_MCP_URL=
|
||||
DEFAULT_OUTLOOK_MCP_SERVER_ID=outlook_mcp
|
||||
|
||||
DEPLOY_PUBLIC_SCHEME=http
|
||||
DEPLOY_PUBLIC_BASE_DOMAIN=203.0.113.10.nip.io
|
||||
DEPLOY_PUBLIC_BASE_DOMAIN=localhost
|
||||
DEPLOY_PUBLIC_HOST_TEMPLATE={slug}.{base_domain}
|
||||
DEPLOY_PUBLIC_PORT=8088
|
||||
DEPLOY_AUTO_START_PROXY=1
|
||||
@ -26,5 +26,5 @@ DEPLOY_HEALTH_TIMEOUT_SECONDS=60
|
||||
DEPLOY_HEALTH_INTERVAL_SECONDS=1
|
||||
|
||||
# Passed through to create-instance.sh when the app-instance image is rebuilt.
|
||||
AUTH_PORTAL_URL=http://203.0.113.10:3081
|
||||
AUTH_PORTAL_URL=http://127.0.0.1:3081
|
||||
AUTH_PORTAL_PORT=3081
|
||||
|
||||
@ -32,7 +32,7 @@
|
||||
默认实例 URL 形如：
|
||||
|
||||
```text
|
||||
http://<instance-slug>.127.0.0.1.nip.io:8088
|
||||
http://<instance-slug>.localhost:8088
|
||||
```
|
||||
|
||||
实例容器本身的 `20000-29999` 端口默认只绑定到部署机 `127.0.0.1`，外部入口应走 `router-proxy`。
|
||||
|
||||
@ -11,6 +11,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib import error as urllib_error
|
||||
from urllib import parse as urllib_parse
|
||||
from urllib import request as urllib_request
|
||||
|
||||
|
||||
@ -37,15 +38,18 @@ API_TOKEN = os.environ.get("DEPLOY_CONTROL_API_TOKEN", "").strip()
|
||||
INSTANCE_IMAGE = os.environ.get("APP_INSTANCE_IMAGE", "beaver/app-instance:latest").strip()
|
||||
INSTANCE_NETWORK_NAME = os.environ.get("APP_INSTANCE_NETWORK_NAME", "beaver-instance-edge").strip()
|
||||
DEFAULT_AUTHZ_BASE_URL = os.environ.get("DEFAULT_AUTHZ_BASE_URL", "").strip()
|
||||
DEFAULT_AUTHZ_INTERNAL_TOKEN = os.environ.get("DEFAULT_AUTHZ_INTERNAL_TOKEN", "").strip()
|
||||
DEFAULT_AUTHZ_OUTLOOK_MCP_URL = os.environ.get("DEFAULT_AUTHZ_OUTLOOK_MCP_URL", "").strip()
|
||||
DEFAULT_OUTLOOK_MCP_SERVER_ID = os.environ.get("DEFAULT_OUTLOOK_MCP_SERVER_ID", "outlook_mcp").strip() or "outlook_mcp"
|
||||
DEFAULT_USER_FILES_MAX_UPLOAD_BYTES = os.environ.get("DEFAULT_USER_FILES_MAX_UPLOAD_BYTES", "").strip()
|
||||
PUBLIC_SCHEME = os.environ.get("DEPLOY_PUBLIC_SCHEME", "http").strip() or "http"
|
||||
PUBLIC_BASE_DOMAIN = os.environ.get("DEPLOY_PUBLIC_BASE_DOMAIN", "127.0.0.1.nip.io").strip()
|
||||
PUBLIC_BASE_DOMAIN = os.environ.get("DEPLOY_PUBLIC_BASE_DOMAIN", "localhost").strip()
|
||||
PUBLIC_HOST_TEMPLATE = os.environ.get("DEPLOY_PUBLIC_HOST_TEMPLATE", "{slug}.{base_domain}").strip()
|
||||
PUBLIC_PORT = int(os.environ.get("DEPLOY_PUBLIC_PORT", "8088").strip() or "8088")
|
||||
AUTO_START_PROXY = os.environ.get("DEPLOY_AUTO_START_PROXY", "1").strip() not in {"0", "false", "False"}
|
||||
HEALTH_TIMEOUT_SECONDS = float(os.environ.get("DEPLOY_HEALTH_TIMEOUT_SECONDS", "60").strip() or "60")
|
||||
HEALTH_INTERVAL_SECONDS = float(os.environ.get("DEPLOY_HEALTH_INTERVAL_SECONDS", "1").strip() or "1")
|
||||
UPSTREAM_TIMEOUT_SECONDS = float(os.environ.get("DEPLOY_UPSTREAM_TIMEOUT_SECONDS", "90").strip() or "90")
|
||||
INSTANCE_INTERNAL_PORT = int(os.environ.get("APP_INSTANCE_INTERNAL_PORT", "8080").strip() or "8080")
|
||||
SERVER_HOST = os.environ.get("DEPLOY_CONTROL_HOST", "0.0.0.0").strip() or "0.0.0.0"
|
||||
SERVER_PORT = int(os.environ.get("DEPLOY_CONTROL_PORT", "8090").strip() or "8090")
|
||||
@ -263,9 +267,13 @@ def create_or_get_instance(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
]
|
||||
if authz_base_url:
|
||||
command.extend(["--authz-base-url", authz_base_url])
|
||||
if DEFAULT_AUTHZ_INTERNAL_TOKEN:
|
||||
command.extend(["--authz-internal-token", DEFAULT_AUTHZ_INTERNAL_TOKEN])
|
||||
if authz_outlook_mcp_url:
|
||||
command.extend(["--authz-outlook-mcp-url", authz_outlook_mcp_url])
|
||||
command.extend(["--outlook-mcp-server-id", DEFAULT_OUTLOOK_MCP_SERVER_ID])
|
||||
if DEFAULT_USER_FILES_MAX_UPLOAD_BYTES:
|
||||
command.extend(["--user-files-max-upload-bytes", DEFAULT_USER_FILES_MAX_UPLOAD_BYTES])
|
||||
if payload.get("replace") is True:
|
||||
command.append("--replace")
|
||||
|
||||
@ -498,21 +506,108 @@ def resolve_instance(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def remove_instance(instance_id: str, purge_data: bool) -> dict[str, Any]:
|
||||
def deprovision_user_files(
|
||||
*,
|
||||
backend_id: str,
|
||||
authz_base_url: str,
|
||||
best_effort: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
if not backend_id:
|
||||
return {"ok": False, "status": "failed", "error": "backend_id is missing"}
|
||||
if not authz_base_url:
|
||||
return {"ok": False, "status": "failed", "error": "AuthZ base URL is not configured"}
|
||||
if not DEFAULT_AUTHZ_INTERNAL_TOKEN:
|
||||
return {"ok": False, "status": "failed", "error": "AuthZ internal token is not configured"}
|
||||
|
||||
query = urllib_parse.urlencode({"best_effort": "1" if best_effort else "0"})
|
||||
quoted_backend_id = urllib_parse.quote(backend_id, safe="")
|
||||
url = f"{authz_base_url.rstrip('/')}/internal/backends/{quoted_backend_id}/user-files?{query}"
|
||||
request = urllib_request.Request(
|
||||
url,
|
||||
method="DELETE",
|
||||
headers={
|
||||
"Authorization": f"Bearer {DEFAULT_AUTHZ_INTERNAL_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib_request.urlopen(request, timeout=UPSTREAM_TIMEOUT_SECONDS) as response:
|
||||
raw = response.read().decode("utf-8")
|
||||
except urllib_error.HTTPError as exc:
|
||||
detail = exc.reason
|
||||
try:
|
||||
payload = json.loads(exc.read().decode("utf-8"))
|
||||
if isinstance(payload, dict):
|
||||
detail = str(payload.get("detail") or detail)
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": False, "status": "failed", "error": detail, "status_code": exc.code}
|
||||
except (urllib_error.URLError, TimeoutError) as exc:
|
||||
return {"ok": False, "status": "failed", "error": str(exc)}
|
||||
|
||||
if not raw.strip():
|
||||
return {"ok": True, "status": "removed"}
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return {"ok": False, "status": "failed", "error": "AuthZ response was not valid JSON"}
|
||||
if not isinstance(payload, dict):
|
||||
return {"ok": False, "status": "failed", "error": "AuthZ response must be a JSON object"}
|
||||
payload.setdefault("ok", True)
|
||||
return payload
|
||||
|
||||
|
||||
def remove_instance(instance_id: str, purge_data: bool, purge_user_files: bool = False) -> dict[str, Any]:
|
||||
if not instance_id.strip():
|
||||
raise ApiError(HTTPStatus.BAD_REQUEST, "instance id is required")
|
||||
record = get_registry_record(instance_id=instance_id)
|
||||
if record is None:
|
||||
local_result: dict[str, Any] = {
|
||||
"instance_id": instance_id,
|
||||
"status": "already_absent",
|
||||
"already_absent": True,
|
||||
}
|
||||
user_files_result: dict[str, Any] = {"ok": True, "status": "skipped"}
|
||||
if purge_user_files:
|
||||
user_files_result = deprovision_user_files(
|
||||
backend_id=instance_id,
|
||||
authz_base_url=DEFAULT_AUTHZ_BASE_URL,
|
||||
best_effort=True,
|
||||
)
|
||||
return {
|
||||
"ok": not purge_user_files or bool(user_files_result.get("ok")),
|
||||
"instance": local_result,
|
||||
"local": local_result,
|
||||
"user_files": user_files_result,
|
||||
}
|
||||
backend_id = str(record.get("backend_id", "") or record.get("username", "") or instance_id).strip()
|
||||
authz_base_url = str(record.get("authz_base_url", "") or DEFAULT_AUTHZ_BASE_URL).strip()
|
||||
|
||||
command = [str(REMOVE_INSTANCE_SCRIPT), "--instance-id", instance_id]
|
||||
if purge_data:
|
||||
command.append("--purge-data")
|
||||
output = run_command(command, cwd=APP_INSTANCE_DIR)
|
||||
ensure_proxy()
|
||||
result: dict[str, str] = {}
|
||||
local_result: dict[str, str] = {}
|
||||
for line in output.splitlines():
|
||||
if "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
result[key] = value
|
||||
return result
|
||||
local_result[key] = value
|
||||
|
||||
user_files_result: dict[str, Any] = {"ok": True, "status": "skipped"}
|
||||
if purge_user_files:
|
||||
user_files_result = deprovision_user_files(
|
||||
backend_id=backend_id,
|
||||
authz_base_url=authz_base_url,
|
||||
best_effort=True,
|
||||
)
|
||||
return {
|
||||
"ok": bool(local_result) and (not purge_user_files or bool(user_files_result.get("ok"))),
|
||||
"instance": local_result,
|
||||
"local": local_result,
|
||||
"user_files": user_files_result,
|
||||
}
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
@ -587,11 +682,17 @@ class Handler(BaseHTTPRequestHandler):
|
||||
def do_DELETE(self) -> None: # noqa: N802
|
||||
try:
|
||||
self._require_auth()
|
||||
if not self.path.startswith("/api/instances/"):
|
||||
parsed = urllib_parse.urlparse(self.path)
|
||||
if not parsed.path.startswith("/api/instances/"):
|
||||
raise ApiError(HTTPStatus.NOT_FOUND, "not found")
|
||||
instance_id = self.path.rsplit("/", 1)[-1]
|
||||
instance_id = urllib_parse.unquote(parsed.path.rsplit("/", 1)[-1])
|
||||
query = urllib_parse.parse_qs(parsed.query)
|
||||
purge_data = self.headers.get("X-Purge-Data", "").strip() == "1"
|
||||
self._json_response(HTTPStatus.OK, remove_instance(instance_id, purge_data))
|
||||
purge_user_files = (
|
||||
self.headers.get("X-Purge-User-Files", "").strip() == "1"
|
||||
or query.get("purge_user_files", [""])[-1] in {"1", "true", "True", "yes"}
|
||||
)
|
||||
self._json_response(HTTPStatus.OK, remove_instance(instance_id, purge_data, purge_user_files))
|
||||
except ApiError as exc:
|
||||
self._json_response(exc.status_code, {"detail": exc.detail})
|
||||
|
||||
|
||||
148
deploy-control/tests/test_delete_orchestration.py
Normal file
148
deploy-control/tests/test_delete_orchestration.py
Normal file
@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SERVER_PATH = Path(__file__).resolve().parents[1] / "server.py"
|
||||
|
||||
|
||||
def _load_server_module():
|
||||
spec = importlib.util.spec_from_file_location("deploy_control_server_for_tests", SERVER_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _record() -> dict[str, Any]:
|
||||
return {
|
||||
"instance_id": "smoke-user",
|
||||
"username": "smoke-user",
|
||||
"backend_id": "smoke-backend",
|
||||
"authz_base_url": "http://authz.local",
|
||||
}
|
||||
|
||||
|
||||
def test_remove_instance_without_user_file_purge_skips_authz(monkeypatch) -> None:
|
||||
server = _load_server_module()
|
||||
calls: list[Any] = []
|
||||
|
||||
monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: _record())
|
||||
monkeypatch.setattr(
|
||||
server,
|
||||
"run_command",
|
||||
lambda *_args, **_kwargs: "instance_id=smoke-user\npurged_data=1",
|
||||
)
|
||||
monkeypatch.setattr(server, "ensure_proxy", lambda: None)
|
||||
monkeypatch.setattr(server, "deprovision_user_files", lambda **kwargs: calls.append(kwargs))
|
||||
|
||||
result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=False)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["local"]["instance_id"] == "smoke-user"
|
||||
assert result["user_files"] == {"ok": True, "status": "skipped"}
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_remove_instance_with_user_file_purge_calls_authz_after_resolving_backend(monkeypatch) -> None:
|
||||
server = _load_server_module()
|
||||
calls: list[dict[str, Any]] = []
|
||||
|
||||
monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: _record())
|
||||
monkeypatch.setattr(
|
||||
server,
|
||||
"run_command",
|
||||
lambda *_args, **_kwargs: "instance_id=smoke-user\npurged_data=1",
|
||||
)
|
||||
monkeypatch.setattr(server, "ensure_proxy", lambda: None)
|
||||
|
||||
def fake_deprovision(**kwargs: Any) -> dict[str, Any]:
|
||||
calls.append(kwargs)
|
||||
return {"ok": True, "objects": {"status": "removed", "deleted": 1}}
|
||||
|
||||
monkeypatch.setattr(server, "deprovision_user_files", fake_deprovision)
|
||||
|
||||
result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=True)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert calls == [
|
||||
{
|
||||
"backend_id": "smoke-backend",
|
||||
"authz_base_url": "http://authz.local",
|
||||
"best_effort": True,
|
||||
}
|
||||
]
|
||||
assert result["user_files"]["objects"] == {"status": "removed", "deleted": 1}
|
||||
|
||||
|
||||
def test_remove_instance_reports_user_file_cleanup_failure_separately(monkeypatch) -> None:
|
||||
server = _load_server_module()
|
||||
|
||||
monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: _record())
|
||||
monkeypatch.setattr(
|
||||
server,
|
||||
"run_command",
|
||||
lambda *_args, **_kwargs: "instance_id=smoke-user\npurged_data=1",
|
||||
)
|
||||
monkeypatch.setattr(server, "ensure_proxy", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
server,
|
||||
"deprovision_user_files",
|
||||
lambda **_kwargs: {"ok": False, "status": "failed", "error": "AuthZ unavailable"},
|
||||
)
|
||||
|
||||
result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=True)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["local"]["instance_id"] == "smoke-user"
|
||||
assert result["user_files"] == {"ok": False, "status": "failed", "error": "AuthZ unavailable"}
|
||||
|
||||
|
||||
def test_remove_already_absent_instance_is_idempotent_without_file_purge(monkeypatch) -> None:
|
||||
server = _load_server_module()
|
||||
calls: list[Any] = []
|
||||
|
||||
monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: None)
|
||||
monkeypatch.setattr(server, "run_command", lambda *_args, **_kwargs: calls.append(_args))
|
||||
monkeypatch.setattr(server, "deprovision_user_files", lambda **kwargs: calls.append(kwargs))
|
||||
|
||||
result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=False)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["local"] == {
|
||||
"instance_id": "smoke-user",
|
||||
"status": "already_absent",
|
||||
"already_absent": True,
|
||||
}
|
||||
assert result["user_files"] == {"ok": True, "status": "skipped"}
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_remove_already_absent_instance_can_retry_user_file_cleanup(monkeypatch) -> None:
|
||||
server = _load_server_module()
|
||||
calls: list[dict[str, Any]] = []
|
||||
monkeypatch.setattr(server, "DEFAULT_AUTHZ_BASE_URL", "http://authz.local")
|
||||
|
||||
monkeypatch.setattr(server, "get_registry_record", lambda **_kwargs: None)
|
||||
monkeypatch.setattr(server, "run_command", lambda *_args, **_kwargs: "should-not-run")
|
||||
|
||||
def fake_deprovision(**kwargs: Any) -> dict[str, Any]:
|
||||
calls.append(kwargs)
|
||||
return {"ok": True, "settings_found": False, "objects": {"status": "absent"}}
|
||||
|
||||
monkeypatch.setattr(server, "deprovision_user_files", fake_deprovision)
|
||||
|
||||
result = server.remove_instance("smoke-user", purge_data=True, purge_user_files=True)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["local"]["already_absent"] is True
|
||||
assert calls == [
|
||||
{
|
||||
"backend_id": "smoke-user",
|
||||
"authz_base_url": "http://authz.local",
|
||||
"best_effort": True,
|
||||
}
|
||||
]
|
||||
assert result["user_files"]["settings_found"] is False
|
||||
Reference in New Issue
Block a user