feat: integrate MinIO-backed user filesystem

authz-service/src/app/json_store.py

@@ -7,7 +7,15 @@ from pathlib import Path
 from threading import Lock
 from typing import Any
 
-from app.models import ChannelSettings, BackendCredential, BackendRecord, OutlookSettings, UserRecord, utcnow_iso
+from app.models import (
+    ChannelSettings,
+    BackendCredential,
+    BackendRecord,
+    MinIOSettings,
+    OutlookSettings,
+    UserRecord,
+    utcnow_iso,
+)
 
 
 class JsonStore:
@@ -152,6 +160,19 @@ class JsonStore:
             return None
         return OutlookSettings.model_validate(outlook)
 
+    def get_minio_settings(self, backend_id: str) -> MinIOSettings | None:
+        raw = self._read_json(self.settings_path, {"settings": {}})
+        root = raw.get("settings", {})
+        if not isinstance(root, dict):
+            return None
+        backend_settings = root.get(backend_id, {})
+        if not isinstance(backend_settings, dict):
+            return None
+        minio = backend_settings.get("minio")
+        if not isinstance(minio, dict):
+            return None
+        return MinIOSettings.model_validate(minio)
+
     def list_channel_settings(self, backend_id: str) -> dict[str, ChannelSettings]:
         raw = self._read_json(self.settings_path, {"settings": {}})
         root = raw.get("settings", {})
@@ -185,6 +206,19 @@ class JsonStore:
         self._write_json(self.settings_path, {"settings": root})
         return backend_settings
 
+    def save_minio_settings(self, backend_id: str, settings: MinIOSettings) -> dict[str, Any]:
+        raw = self._read_json(self.settings_path, {"settings": {}})
+        root = raw.get("settings", {})
+        if not isinstance(root, dict):
+            root = {}
+        backend_settings = root.get(backend_id, {})
+        if not isinstance(backend_settings, dict):
+            backend_settings = {}
+        backend_settings["minio"] = settings.model_dump(mode="json")
+        root[backend_id] = backend_settings
+        self._write_json(self.settings_path, {"settings": root})
+        return backend_settings
+
     def save_channel_settings(self, backend_id: str, channel_id: str, settings: ChannelSettings) -> dict[str, Any]:
         raw = self._read_json(self.settings_path, {"settings": {}})
         root = raw.get("settings", {})
@@ -218,6 +252,22 @@ class JsonStore:
         self._write_json(self.settings_path, {"settings": root})
         return True
 
+    def delete_minio_settings(self, backend_id: str) -> bool:
+        raw = self._read_json(self.settings_path, {"settings": {}})
+        root = raw.get("settings", {})
+        if not isinstance(root, dict):
+            return False
+        backend_settings = root.get(backend_id)
+        if not isinstance(backend_settings, dict) or "minio" not in backend_settings:
+            return False
+        backend_settings.pop("minio", None)
+        if backend_settings:
+            root[backend_id] = backend_settings
+        else:
+            root.pop(backend_id, None)
+        self._write_json(self.settings_path, {"settings": root})
+        return True
+
     def delete_channel_settings(self, backend_id: str, channel_id: str) -> bool:
         raw = self._read_json(self.settings_path, {"settings": {}})
         root = raw.get("settings", {})

authz-service/src/app/main.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
+import asyncio
 import os
 import re
 from pathlib import Path
 from typing import Any
 
 import httpx
-from fastapi import Depends, FastAPI, Header, HTTPException, Request
+from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request
 from fastapi.responses import JSONResponse
 
 from app.json_store import JsonStore
@@ -16,6 +17,7 @@ from app.models import (
     BackendRecord,
     IntrospectRequest,
     IntrospectResponse,
+    MinIOSettings,
     OAuthTokenRequest,
     OAuthTokenResponse,
     OutlookSettings,
@@ -30,6 +32,12 @@ from app.models import (
     UserRecord,
     utcnow_iso,
 )
+from app.minio_provisioning import (
+    MinIODeprovisioningError,
+    MinIOProvisioningError,
+    deprovision_user_file_minio_resources,
+    provision_user_file_minio_settings,
+)
 from app.security import JwtSigner, generate_client_secret, hash_secret, verify_secret
 
 DATA_DIR = Path(os.getenv("AUTHZ_DATA_DIR", Path(__file__).resolve().parents[1] / "data"))
@@ -360,6 +368,42 @@ def _upsert_user(
     return user
 
 
+async def _ensure_user_file_storage_settings(backend_id: str) -> MinIOSettings | None:
+    try:
+        settings = await asyncio.to_thread(
+            provision_user_file_minio_settings,
+            backend_id=backend_id,
+            existing=store.get_minio_settings(backend_id),
+        )
+    except MinIOProvisioningError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+    if settings is not None:
+        store.save_minio_settings(backend_id, settings)
+        store.touch_backend(backend_id)
+    return settings
+
+
+async def _deprovision_user_file_storage(backend_id: str, *, best_effort: bool = False) -> dict[str, Any]:
+    existing = store.get_minio_settings(backend_id)
+    try:
+        result = await asyncio.to_thread(
+            deprovision_user_file_minio_resources,
+            backend_id=backend_id,
+            existing=existing,
+            best_effort=best_effort,
+        )
+    except MinIODeprovisioningError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+
+    settings_removed = False
+    if result.get("ok") is True:
+        settings_removed = store.delete_minio_settings(backend_id)
+        if settings_removed:
+            store.touch_backend(backend_id)
+    result["settings"] = {"status": "removed" if settings_removed else "absent"}
+    return result
+
+
 def _resolve_register_backend_payload(req: RegisterBackendRequest) -> tuple[str, str, str, str | None]:
     backend_name = req.name.strip() or "backend"
     backend_id = _clean_optional(req.backend_id)
@@ -475,6 +519,7 @@ async def register_backend(req: RegisterBackendRequest) -> RegisterBackendRespon
         base_url=base_url,
         frontend_base_url=frontend_base_url,
     )
+    await _ensure_user_file_storage_settings(backend.backend_id)
     return RegisterBackendResponse(
         backend_id=backend.backend_id,
         client_id=backend.backend_id,
@@ -589,6 +634,39 @@ async def delete_outlook_settings(backend_id: str) -> dict[str, Any]:
     return {"ok": removed}
 
 
+@app.get("/backends/{backend_id}/settings/minio")
+async def get_minio_settings(backend_id: str) -> dict[str, Any]:
+    _ensure_backend(backend_id)
+    settings = store.get_minio_settings(backend_id)
+    if settings is None:
+        return {"configured": False}
+    return settings.masked_dict()
+
+
+@app.post("/backends/{backend_id}/settings/minio")
+async def save_minio_settings(backend_id: str, payload: MinIOSettings) -> dict[str, Any]:
+    _ensure_backend(backend_id)
+    existing = store.get_minio_settings(backend_id)
+    if not payload.secret_key:
+        if existing is None or not existing.secret_key:
+            raise HTTPException(status_code=400, detail="Secret key is required for initial MinIO setup")
+        payload = payload.model_copy(update={"secret_key": existing.secret_key, "updated_at": utcnow_iso()})
+    else:
+        payload = payload.model_copy(update={"updated_at": utcnow_iso()})
+    store.save_minio_settings(backend_id, payload)
+    store.touch_backend(backend_id)
+    return payload.masked_dict()
+
+
+@app.delete("/backends/{backend_id}/settings/minio")
+async def delete_minio_settings(backend_id: str) -> dict[str, Any]:
+    _ensure_backend(backend_id)
+    removed = store.delete_minio_settings(backend_id)
+    if removed:
+        store.touch_backend(backend_id)
+    return {"ok": removed}
+
+
 @app.get("/backends/{backend_id}/settings/channels")
 async def list_channel_settings(backend_id: str) -> dict[str, Any]:
     _ensure_backend(backend_id)
@@ -662,6 +740,7 @@ async def oauth_register(req: RegisterUserRequest) -> RegisterUserResponse:
         email=_clean_optional(req.email),
         default_backend_id=backend.backend_id,
     )
+    await _ensure_user_file_storage_settings(backend.backend_id)
     return RegisterUserResponse(
         user=user,
         backend=RegisterUserBackendResult(
@@ -686,6 +765,23 @@ async def get_internal_outlook_settings(backend_id: str) -> dict[str, Any]:
     return settings.model_dump(mode="json")
 
 
+@app.get("/internal/backends/{backend_id}/settings/minio", dependencies=[Depends(_require_internal)])
+async def get_internal_minio_settings(backend_id: str) -> dict[str, Any]:
+    _ensure_backend(backend_id)
+    settings = store.get_minio_settings(backend_id)
+    if settings is None:
+        raise HTTPException(status_code=404, detail="MinIO settings not configured")
+    return settings.model_dump(mode="json")
+
+
+@app.delete("/internal/backends/{backend_id}/user-files", dependencies=[Depends(_require_internal)])
+async def delete_internal_user_files(
+    backend_id: str,
+    best_effort: bool = Query(default=False),
+) -> dict[str, Any]:
+    return await _deprovision_user_file_storage(backend_id, best_effort=best_effort)
+
+
 @app.get("/internal/backends/{backend_id}/settings/channels", dependencies=[Depends(_require_internal)])
 async def list_internal_channel_settings(backend_id: str) -> dict[str, Any]:
     _ensure_backend(backend_id)

authz-service/src/app/minio_provisioning.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+import os
+import re
+import secrets
+from typing import Any
+
+from .models import MinIOSettings
+
+
+@dataclass(slots=True)
+class MinIOProvisioningConfig:
+    enabled: bool
+    endpoint: str
+    public_endpoint: str
+    admin_access_key: str
+    admin_secret_key: str
+    bucket: str
+    secure: bool
+    region: str | None
+
+
+class MinIOProvisioningError(RuntimeError):
+    pass
+
+
+class MinIODeprovisioningError(RuntimeError):
+    pass
+
+
+def minio_provisioning_config_from_env() -> MinIOProvisioningConfig:
+    endpoint = _env("USER_FILES_MINIO_ENDPOINT") or _env("MINIO_ENDPOINT")
+    public_endpoint = _env("USER_FILES_MINIO_PUBLIC_ENDPOINT") or endpoint
+    return MinIOProvisioningConfig(
+        enabled=_truthy(_env("USER_FILES_MINIO_PROVISIONING_ENABLED")),
+        endpoint=endpoint,
+        public_endpoint=public_endpoint,
+        admin_access_key=_env("USER_FILES_MINIO_ADMIN_ACCESS_KEY") or _env("MINIO_ROOT_USER"),
+        admin_secret_key=_env("USER_FILES_MINIO_ADMIN_SECRET_KEY") or _env("MINIO_ROOT_PASSWORD"),
+        bucket=_env("USER_FILES_MINIO_BUCKET") or "beaver-user-files",
+        secure=_truthy(_env("USER_FILES_MINIO_SECURE")),
+        region=_env("USER_FILES_MINIO_REGION") or None,
+    )
+
+
+def provision_user_file_minio_settings(
+    *,
+    backend_id: str,
+    existing: MinIOSettings | None,
+    config: MinIOProvisioningConfig | None = None,
+) -> MinIOSettings | None:
+    cfg = config or minio_provisioning_config_from_env()
+    if not cfg.enabled:
+        return existing
+    if existing is not None and existing.configured and existing.secret_key:
+        return existing
+    if not cfg.endpoint or not cfg.admin_access_key or not cfg.admin_secret_key:
+        raise MinIOProvisioningError("MinIO provisioning requires endpoint and admin credentials")
+
+    namespace = default_namespace(backend_id)
+    access_key = _access_key_for_backend(backend_id)
+    secret_key = secrets.token_urlsafe(32)
+    policy_name = _policy_name_for_backend(backend_id)
+
+    try:
+        from minio import Minio
+        from minio import MinioAdmin
+        from minio.credentials import StaticProvider
+
+        client = Minio(
+            endpoint=cfg.endpoint,
+            access_key=cfg.admin_access_key,
+            secret_key=cfg.admin_secret_key,
+            secure=cfg.secure,
+            region=cfg.region,
+        )
+        if not client.bucket_exists(cfg.bucket):
+            client.make_bucket(cfg.bucket, location=cfg.region)
+
+        admin = MinioAdmin(
+            endpoint=cfg.endpoint,
+            credentials=StaticProvider(
+                access_key=cfg.admin_access_key,
+                secret_key=cfg.admin_secret_key,
+            ),
+            secure=cfg.secure,
+        )
+        try:
+            admin.user_add(access_key, secret_key)
+        except Exception:
+            # Treat an existing user as idempotent; keep the newly generated
+            # secret only when user creation succeeded.
+            if existing is not None and existing.secret_key:
+                secret_key = existing.secret_key
+            else:
+                raise
+
+        policy = _namespace_policy(bucket=cfg.bucket, namespace=namespace)
+        admin.policy_add(policy_name, policy=policy)
+        admin.attach_policy(policies=[policy_name], user=access_key)
+    except Exception as exc:
+        raise MinIOProvisioningError(f"MinIO user file provisioning failed: {exc}") from exc
+
+    return MinIOSettings(
+        endpoint=cfg.public_endpoint,
+        access_key=access_key,
+        secret_key=secret_key,
+        bucket=cfg.bucket,
+        namespace=namespace,
+        secure=cfg.secure,
+        region=cfg.region,
+    )
+
+
+def deprovision_user_file_minio_resources(
+    *,
+    backend_id: str,
+    existing: MinIOSettings | None,
+    best_effort: bool = False,
+    config: MinIOProvisioningConfig | None = None,
+) -> dict[str, Any]:
+    cfg = config or minio_provisioning_config_from_env()
+    settings_found = existing is not None
+    result: dict[str, Any] = {
+        "ok": True,
+        "backend_id": backend_id,
+        "settings_found": settings_found,
+        "best_effort": best_effort,
+        "bucket": "skipped",
+        "objects": {"status": "absent", "deleted": 0},
+        "user": {"status": "absent"},
+        "policy": {"status": "absent"},
+    }
+
+    if existing is None and not best_effort:
+        return result
+
+    bucket = (existing.bucket if existing else None) or cfg.bucket
+    namespace = (existing.namespace if existing else None) or default_namespace(backend_id)
+    access_key = (existing.access_key if existing else None) or _access_key_for_backend(backend_id)
+    policy_name = _policy_name_for_backend(backend_id)
+
+    if not bucket or not namespace.strip("/"):
+        raise MinIODeprovisioningError("MinIO deprovisioning requires bucket and namespace")
+    if not cfg.endpoint or not cfg.admin_access_key or not cfg.admin_secret_key:
+        raise MinIODeprovisioningError("MinIO deprovisioning requires endpoint and admin credentials")
+
+    try:
+        from minio import Minio
+        from minio import MinioAdmin
+        from minio.credentials import StaticProvider
+        from minio.deleteobjects import DeleteObject
+
+        client = Minio(
+            endpoint=cfg.endpoint,
+            access_key=cfg.admin_access_key,
+            secret_key=cfg.admin_secret_key,
+            secure=cfg.secure,
+            region=cfg.region,
+        )
+        admin = MinioAdmin(
+            endpoint=cfg.endpoint,
+            credentials=StaticProvider(
+                access_key=cfg.admin_access_key,
+                secret_key=cfg.admin_secret_key,
+            ),
+            secure=cfg.secure,
+        )
+
+        prefix = namespace.strip("/")
+        if client.bucket_exists(bucket):
+            result["bucket"] = "present"
+            objects = [
+                DeleteObject(item.object_name)
+                for item in client.list_objects(bucket, prefix=f"{prefix}/", recursive=True)
+                if getattr(item, "object_name", None)
+            ]
+            errors = list(client.remove_objects(bucket, objects)) if objects else []
+            if errors:
+                result["objects"] = {
+                    "status": "failed",
+                    "deleted": 0,
+                    "errors": [_safe_error_text(error) for error in errors],
+                }
+                result["ok"] = False
+            else:
+                result["objects"] = {
+                    "status": "removed" if objects else "absent",
+                    "deleted": len(objects),
+                }
+        else:
+            result["bucket"] = "absent"
+            result["objects"] = {"status": "absent", "deleted": 0}
+
+        result["policy_detach"] = _admin_step(
+            lambda: _call_admin_method(
+                admin,
+                ["detach_policy"],
+                policies=[policy_name],
+                user=access_key,
+            )
+        )
+        result["user"] = _admin_step(lambda: _call_admin_method(admin, ["user_remove", "remove_user"], access_key))
+        result["policy"] = _admin_step(
+            lambda: _call_admin_method(admin, ["policy_remove", "remove_policy"], policy_name)
+        )
+    except Exception as exc:
+        raise MinIODeprovisioningError(f"MinIO user file deprovisioning failed: {_safe_error_text(exc)}") from exc
+
+    result["ok"] = bool(result["ok"]) and all(
+        item.get("status") != "failed"
+        for item in (
+            result["objects"],
+            result["policy_detach"],
+            result["user"],
+            result["policy"],
+        )
+        if isinstance(item, dict)
+    )
+    return result
+
+
+def default_namespace(backend_id: str) -> str:
+    return f"users/{backend_id.strip().strip('/')}"
+
+
+def _namespace_policy(*, bucket: str, namespace: str) -> dict[str, object]:
+    prefix = namespace.strip("/")
+    return {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetBucketLocation"],
+                "Resource": [f"arn:aws:s3:::{bucket}"],
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:ListBucket"],
+                "Resource": [f"arn:aws:s3:::{bucket}"],
+                "Condition": {"StringLike": {"s3:prefix": [f"{prefix}/*", prefix]}},
+            },
+            {
+                "Effect": "Allow",
+                "Action": [
+                    "s3:GetObject",
+                    "s3:PutObject",
+                    "s3:DeleteObject",
+                ],
+                "Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*"],
+            },
+        ],
+    }
+
+
+def policy_json_for_backend(*, backend_id: str, bucket: str = "beaver-user-files") -> str:
+    return json.dumps(_namespace_policy(bucket=bucket, namespace=default_namespace(backend_id)), indent=2)
+
+
+def _access_key_for_backend(backend_id: str) -> str:
+    slug = re.sub(r"[^A-Za-z0-9_-]+", "-", backend_id.strip()).strip("-_")
+    if not slug:
+        slug = secrets.token_hex(6)
+    return f"beaver-{slug}"[:64]
+
+
+def _policy_name_for_backend(backend_id: str) -> str:
+    slug = re.sub(r"[^A-Za-z0-9_-]+", "-", backend_id.strip()).strip("-_")
+    return f"beaver-user-files-{slug or secrets.token_hex(6)}"[:128]
+
+
+def _call_admin_method(admin: object, names: list[str], *args: Any, **kwargs: Any) -> Any:
+    for name in names:
+        method = getattr(admin, name, None)
+        if callable(method):
+            return method(*args, **kwargs)
+    raise AttributeError(f"MinIO admin client does not support any of: {', '.join(names)}")
+
+
+def _admin_step(action: Any) -> dict[str, Any]:
+    try:
+        action()
+    except Exception as exc:
+        if _is_absent_error(exc):
+            return {"status": "absent"}
+        return {"status": "failed", "error": _safe_error_text(exc)}
+    return {"status": "removed"}
+
+
+def _is_absent_error(exc: Exception) -> bool:
+    text = _safe_error_text(exc).lower()
+    absent_markers = (
+        "not found",
+        "notfound",
+        "no such",
+        "does not exist",
+        "doesn't exist",
+        "specified user does not exist",
+        "specified policy does not exist",
+        "the specified bucket does not exist",
+    )
+    return any(marker in text for marker in absent_markers)
+
+
+def _safe_error_text(exc: object) -> str:
+    text = str(exc).strip()
+    return text or exc.__class__.__name__
+
+
+def _env(name: str) -> str:
+    return os.getenv(name, "").strip()
+
+
+def _truthy(value: str) -> bool:
+    return value.lower() in {"1", "true", "yes", "on"}

authz-service/src/app/models.py

@@ -77,6 +77,24 @@ class OutlookSettings(BaseModel):
         return data
 
 
+class MinIOSettings(BaseModel):
+    configured: bool = True
+    endpoint: str
+    access_key: str
+    secret_key: str
+    bucket: str | None = None
+    namespace: str | None = None
+    secure: bool = False
+    region: str | None = None
+    updated_at: str = Field(default_factory=utcnow_iso)
+
+    def masked_dict(self) -> dict[str, Any]:
+        data = self.model_dump()
+        data.pop("secret_key", None)
+        data["secret_key_masked"] = True
+        return data
+
+
 class BackendRoutingPayload(BaseModel):
     name: str | None = None
     backend_id: str | None = None