replace main with lightweight memory gateway
This commit is contained in:
7
core/__init__.py
Normal file
7
core/__init__.py
Normal file
@ -0,0 +1,7 @@
|
||||
"""Lightweight Memory Gateway for EverOS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["__version__"]
|
||||
|
||||
__version__ = "0.1.0"
|
||||
167
core/api.py
Normal file
167
core/api.py
Normal file
@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, FastAPI, File, Form, HTTPException, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .config import GatewayConfig
|
||||
from .db import init_db
|
||||
from .everos_client import EverOSClient
|
||||
from .repository import MemoryRepository
|
||||
from .service import MemoryGatewayService
|
||||
|
||||
|
||||
class SearchMemoriesRequest(BaseModel):
|
||||
user_id: str = Field(min_length=1)
|
||||
user_key: str = Field(min_length=1)
|
||||
conversation_id: str | None = None
|
||||
query: str = Field(min_length=1)
|
||||
scope: list[Literal["current_chat", "resources", "all_user_memory"]] = Field(
|
||||
default_factory=lambda: ["current_chat", "resources"]
|
||||
)
|
||||
top_k: int = Field(default=8, ge=1, le=100)
|
||||
app_id: str = "default"
|
||||
project_id: str = "default"
|
||||
|
||||
|
||||
class MemoryOverrideRequest(BaseModel):
|
||||
user_id: str = Field(min_length=1)
|
||||
user_key: str = Field(min_length=1)
|
||||
session_id: str | None = None
|
||||
override_text: str = Field(min_length=1)
|
||||
|
||||
|
||||
class MemoryDeleteRequest(BaseModel):
|
||||
user_id: str = Field(min_length=1)
|
||||
user_key: str = Field(min_length=1)
|
||||
session_id: str | None = None
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
class UserCreateRequest(BaseModel):
|
||||
user_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
def create_app(
|
||||
*,
|
||||
config: GatewayConfig | None = None,
|
||||
everos_client: Any | None = None,
|
||||
) -> FastAPI:
|
||||
cfg = config or GatewayConfig.from_env()
|
||||
init_db(cfg.database_path)
|
||||
repository = MemoryRepository(cfg.database_path)
|
||||
client = everos_client or EverOSClient(cfg.everos_base_url)
|
||||
service = MemoryGatewayService(cfg, repository, client)
|
||||
|
||||
app = FastAPI(title="memory-gateway2", version="0.1.0")
|
||||
app.state.config = cfg
|
||||
app.state.repository = repository
|
||||
app.state.everos_client = client
|
||||
app.state.gateway_service = service
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def require_user(user_id: str, user_key: str) -> None:
|
||||
if not service.authenticate_user(user_id, user_key):
|
||||
raise HTTPException(status_code=401, detail="invalid user credentials")
|
||||
|
||||
@router.post("/users")
|
||||
async def create_user(request: UserCreateRequest) -> dict[str, Any]:
|
||||
return service.create_user(request.user_id)
|
||||
|
||||
@router.post("/resources")
|
||||
async def upload_resource(
|
||||
user_id: str = Form(...),
|
||||
user_key: str = Form(...),
|
||||
app_id: str = Form("default"),
|
||||
project_id: str = Form("default"),
|
||||
title: str | None = Form(None),
|
||||
description: str | None = Form(None),
|
||||
file: UploadFile = File(...),
|
||||
) -> dict[str, Any]:
|
||||
require_user(user_id, user_key)
|
||||
return await service.upload_resource(
|
||||
user_id=user_id,
|
||||
app_id=app_id,
|
||||
project_id=project_id,
|
||||
file=file,
|
||||
title=title,
|
||||
description=description,
|
||||
)
|
||||
|
||||
@router.get("/resources")
|
||||
async def list_resources(
|
||||
user_id: str,
|
||||
user_key: str,
|
||||
) -> dict[str, Any]:
|
||||
require_user(user_id, user_key)
|
||||
return {"resources": service.list_resources(user_id)}
|
||||
|
||||
@router.get("/resources/{resource_id}")
|
||||
async def get_resource(
|
||||
resource_id: str,
|
||||
user_id: str,
|
||||
user_key: str,
|
||||
) -> dict[str, Any]:
|
||||
require_user(user_id, user_key)
|
||||
resource = service.get_resource_detail(resource_id, user_id)
|
||||
if resource is None:
|
||||
return {"resources": []}
|
||||
return {"resources": [resource]}
|
||||
|
||||
@router.delete("/resources/{resource_id}")
|
||||
async def delete_resource(
|
||||
resource_id: str,
|
||||
user_id: str,
|
||||
user_key: str,
|
||||
) -> dict[str, Any]:
|
||||
require_user(user_id, user_key)
|
||||
resource = service.delete_resource(resource_id, user_id)
|
||||
if resource is None:
|
||||
raise HTTPException(status_code=404, detail="resource not found")
|
||||
return resource
|
||||
|
||||
@router.post("/memories/search")
|
||||
async def search_memories(
|
||||
request: SearchMemoriesRequest,
|
||||
) -> dict[str, Any]:
|
||||
require_user(request.user_id, request.user_key)
|
||||
return await service.search_memories(
|
||||
user_id=request.user_id,
|
||||
query=request.query,
|
||||
conversation_id=request.conversation_id,
|
||||
scope=request.scope,
|
||||
top_k=request.top_k,
|
||||
app_id=request.app_id,
|
||||
project_id=request.project_id,
|
||||
)
|
||||
|
||||
@router.patch("/memories/{memory_id}")
|
||||
async def patch_memory(
|
||||
memory_id: str,
|
||||
request: MemoryOverrideRequest,
|
||||
) -> dict[str, Any]:
|
||||
require_user(request.user_id, request.user_key)
|
||||
return service.upsert_override(
|
||||
user_id=request.user_id,
|
||||
memory_id=memory_id,
|
||||
session_id=request.session_id,
|
||||
override_text=request.override_text,
|
||||
)
|
||||
|
||||
@router.delete("/memories/{memory_id}")
|
||||
async def delete_memory(
|
||||
memory_id: str,
|
||||
request: MemoryDeleteRequest,
|
||||
) -> dict[str, Any]:
|
||||
require_user(request.user_id, request.user_key)
|
||||
return service.delete_memory(
|
||||
user_id=request.user_id,
|
||||
memory_id=memory_id,
|
||||
session_id=request.session_id,
|
||||
reason=request.reason,
|
||||
)
|
||||
|
||||
app.include_router(router)
|
||||
return app
|
||||
40
core/config.py
Normal file
40
core/config.py
Normal file
@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GatewayConfig:
|
||||
everos_base_url: str = "http://127.0.0.1:8000"
|
||||
database_path: Path = _PROJECT_ROOT / "data" / "memory_gateway.sqlite3"
|
||||
storage_dir: Path = _PROJECT_ROOT / "data" / "storage"
|
||||
resource_search_batch_size: int = 50
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> GatewayConfig:
|
||||
return cls(
|
||||
everos_base_url=os.environ.get(
|
||||
"EVEROS_BASE_URL",
|
||||
"http://127.0.0.1:8000",
|
||||
).rstrip("/"),
|
||||
database_path=Path(
|
||||
os.environ.get(
|
||||
"MEMORY_GATEWAY_DB_PATH",
|
||||
str(_PROJECT_ROOT / "data" / "memory_gateway.sqlite3"),
|
||||
)
|
||||
),
|
||||
storage_dir=Path(
|
||||
os.environ.get(
|
||||
"MEMORY_GATEWAY_STORAGE_DIR",
|
||||
str(_PROJECT_ROOT / "data" / "storage"),
|
||||
)
|
||||
),
|
||||
resource_search_batch_size=int(
|
||||
os.environ.get("MEMORY_GATEWAY_RESOURCE_SEARCH_BATCH_SIZE", "50")
|
||||
),
|
||||
)
|
||||
89
core/db.py
Normal file
89
core/db.py
Normal file
@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_key TEXT NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_resources (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
app_id TEXT NOT NULL DEFAULT 'default',
|
||||
project_id TEXT NOT NULL DEFAULT 'default',
|
||||
session_id TEXT NOT NULL,
|
||||
original_filename TEXT,
|
||||
mime_type TEXT,
|
||||
content_type TEXT NOT NULL,
|
||||
uri TEXT NOT NULL,
|
||||
uri_public BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
sha256 TEXT,
|
||||
size_bytes INTEGER,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
error_message TEXT,
|
||||
created_at TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL,
|
||||
deleted_at TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_user_resources_user_scope
|
||||
ON user_resources (user_id, app_id, project_id, status, deleted_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_user_resources_session_id
|
||||
ON user_resources (session_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_user_resources_user_id
|
||||
ON user_resources (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_tombstones (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
memory_id TEXT,
|
||||
session_id TEXT,
|
||||
reason TEXT,
|
||||
created_at TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_tombstones_user_memory
|
||||
ON memory_tombstones (user_id, memory_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_tombstones_user_session
|
||||
ON memory_tombstones (user_id, session_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_overrides (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
memory_id TEXT,
|
||||
session_id TEXT,
|
||||
override_text TEXT NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
created_at TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_overrides_user_memory_active
|
||||
ON memory_overrides (user_id, memory_id, is_active);
|
||||
"""
|
||||
|
||||
|
||||
def connect(db_path: Path) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(db_path: Path) -> None:
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with connect(db_path) as conn:
|
||||
conn.executescript(SCHEMA)
|
||||
41
core/everos_client.py
Normal file
41
core/everos_client.py
Normal file
@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class EverOSClient:
|
||||
def __init__(self, base_url: str, timeout: float = 120.0) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
|
||||
async def add_memory(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return await self._post("/api/v1/memory/add", payload)
|
||||
|
||||
async def flush_memory(
|
||||
self,
|
||||
session_id: str,
|
||||
app_id: str,
|
||||
project_id: str,
|
||||
) -> dict[str, Any]:
|
||||
return await self._post(
|
||||
"/api/v1/memory/flush",
|
||||
{
|
||||
"session_id": session_id,
|
||||
"app_id": app_id,
|
||||
"project_id": project_id,
|
||||
},
|
||||
)
|
||||
|
||||
async def search_memory(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return await self._post("/api/v1/memory/search", payload)
|
||||
|
||||
async def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
async with httpx.AsyncClient(
|
||||
base_url=self.base_url,
|
||||
timeout=self.timeout,
|
||||
) as client:
|
||||
response = await client.post(path, json=payload)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
282
core/repository.py
Normal file
282
core/repository.py
Normal file
@ -0,0 +1,282 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .db import connect
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _row_to_dict(row: Any | None) -> dict[str, Any] | None:
|
||||
if row is None:
|
||||
return None
|
||||
return dict(row)
|
||||
|
||||
|
||||
class MemoryRepository:
|
||||
def __init__(self, db_path: Path) -> None:
|
||||
self.db_path = db_path
|
||||
|
||||
def create_user(self, user_id: str, user_key: str) -> dict[str, Any]:
|
||||
existing = self.get_user(user_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
now = utc_now()
|
||||
with connect(self.db_path) as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO users (id, user_key, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(user_id, user_key, now, now),
|
||||
)
|
||||
conn.commit()
|
||||
user = self.get_user(user_id)
|
||||
if user is None:
|
||||
raise RuntimeError("created user could not be read back")
|
||||
return user
|
||||
|
||||
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||
with connect(self.db_path) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM users WHERE id = ?",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
return _row_to_dict(row)
|
||||
|
||||
def create_resource(self, **values: Any) -> dict[str, Any]:
|
||||
now = utc_now()
|
||||
payload = {
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"deleted_at": None,
|
||||
**values,
|
||||
}
|
||||
columns = ", ".join(payload)
|
||||
placeholders = ", ".join(f":{key}" for key in payload)
|
||||
with connect(self.db_path) as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO user_resources ({columns}) VALUES ({placeholders})",
|
||||
payload,
|
||||
)
|
||||
conn.commit()
|
||||
resource = self.get_resource(str(payload["id"]))
|
||||
if resource is None:
|
||||
raise RuntimeError("created resource could not be read back")
|
||||
return resource
|
||||
|
||||
def update_resource_status(
|
||||
self,
|
||||
resource_id: str,
|
||||
status: str,
|
||||
error_message: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
with connect(self.db_path) as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE user_resources
|
||||
SET status = ?, error_message = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(status, error_message, utc_now(), resource_id),
|
||||
)
|
||||
conn.commit()
|
||||
return self.get_resource(resource_id)
|
||||
|
||||
def soft_delete_resource(
|
||||
self,
|
||||
resource_id: str,
|
||||
user_id: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = utc_now()
|
||||
where = "id = ? AND deleted_at IS NULL"
|
||||
params: tuple[Any, ...] = (now, now, resource_id)
|
||||
if user_id is not None:
|
||||
where += " AND user_id = ?"
|
||||
params = (now, now, resource_id, user_id)
|
||||
with connect(self.db_path) as conn:
|
||||
conn.execute(
|
||||
f"""
|
||||
UPDATE user_resources
|
||||
SET status = 'deleted', deleted_at = ?, updated_at = ?
|
||||
WHERE {where}
|
||||
""",
|
||||
params,
|
||||
)
|
||||
conn.commit()
|
||||
return self.get_resource(resource_id)
|
||||
|
||||
def get_resource(self, resource_id: str) -> dict[str, Any] | None:
|
||||
with connect(self.db_path) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM user_resources WHERE id = ?",
|
||||
(resource_id,),
|
||||
).fetchone()
|
||||
return _row_to_dict(row)
|
||||
|
||||
def get_resource_for_user(
|
||||
self,
|
||||
resource_id: str,
|
||||
user_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
with connect(self.db_path) as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM user_resources
|
||||
WHERE id = ? AND user_id = ? AND deleted_at IS NULL
|
||||
""",
|
||||
(resource_id, user_id),
|
||||
).fetchone()
|
||||
return _row_to_dict(row)
|
||||
|
||||
def get_resource_by_session(self, session_id: str) -> dict[str, Any] | None:
|
||||
with connect(self.db_path) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM user_resources WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
return _row_to_dict(row)
|
||||
|
||||
def get_resource_by_session_for_user(
|
||||
self,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
with connect(self.db_path) as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM user_resources
|
||||
WHERE session_id = ? AND user_id = ? AND deleted_at IS NULL
|
||||
""",
|
||||
(session_id, user_id),
|
||||
).fetchone()
|
||||
return _row_to_dict(row)
|
||||
|
||||
def list_resources(self, user_id: str) -> list[dict[str, Any]]:
|
||||
with connect(self.db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM user_resources
|
||||
WHERE user_id = ? AND deleted_at IS NULL
|
||||
ORDER BY created_at DESC
|
||||
""",
|
||||
(user_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def list_extracted_resources(
|
||||
self,
|
||||
user_id: str,
|
||||
app_id: str,
|
||||
project_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
with connect(self.db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM user_resources
|
||||
WHERE user_id = ?
|
||||
AND app_id = ?
|
||||
AND project_id = ?
|
||||
AND deleted_at IS NULL
|
||||
AND status = 'extracted'
|
||||
ORDER BY created_at DESC
|
||||
""",
|
||||
(user_id, app_id, project_id),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def add_tombstone(
|
||||
self,
|
||||
user_id: str,
|
||||
memory_id: str | None,
|
||||
session_id: str | None,
|
||||
reason: str | None,
|
||||
) -> dict[str, Any]:
|
||||
tombstone_id = f"t_{uuid.uuid4().hex}"
|
||||
with connect(self.db_path) as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO memory_tombstones
|
||||
(id, user_id, memory_id, session_id, reason, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(tombstone_id, user_id, memory_id, session_id, reason, utc_now()),
|
||||
)
|
||||
conn.commit()
|
||||
return {"id": tombstone_id}
|
||||
|
||||
def get_tombstones(self, user_id: str) -> list[dict[str, Any]]:
|
||||
with connect(self.db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM memory_tombstones WHERE user_id = ?",
|
||||
(user_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def upsert_override(
|
||||
self,
|
||||
user_id: str,
|
||||
memory_id: str,
|
||||
session_id: str | None,
|
||||
override_text: str,
|
||||
) -> dict[str, Any]:
|
||||
now = utc_now()
|
||||
with connect(self.db_path) as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT id FROM memory_overrides
|
||||
WHERE user_id = ? AND memory_id = ? AND is_active = TRUE
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(user_id, memory_id),
|
||||
).fetchone()
|
||||
if row:
|
||||
override_id = row["id"]
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE memory_overrides
|
||||
SET session_id = ?, override_text = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(session_id, override_text, now, override_id),
|
||||
)
|
||||
else:
|
||||
override_id = f"o_{uuid.uuid4().hex}"
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO memory_overrides
|
||||
(
|
||||
id, user_id, memory_id, session_id, override_text,
|
||||
is_active, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, TRUE, ?, ?)
|
||||
""",
|
||||
(
|
||||
override_id,
|
||||
user_id,
|
||||
memory_id,
|
||||
session_id,
|
||||
override_text,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return {"id": override_id}
|
||||
|
||||
def get_active_overrides(self, user_id: str) -> list[dict[str, Any]]:
|
||||
with connect(self.db_path) as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM memory_overrides
|
||||
WHERE user_id = ? AND is_active = TRUE
|
||||
""",
|
||||
(user_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
462
core/service.py
Normal file
462
core/service.py
Normal file
@ -0,0 +1,462 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import secrets
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import UploadFile
|
||||
|
||||
from .config import GatewayConfig
|
||||
from .repository import MemoryRepository
|
||||
|
||||
|
||||
def new_resource_id() -> str:
|
||||
return f"r_{uuid.uuid4().hex}"
|
||||
|
||||
|
||||
def resource_session_id(user_id: str, resource_id: str) -> str:
|
||||
return f"resource:{user_id}:{resource_id}"
|
||||
|
||||
|
||||
def public_resource_uri(user_id: str, resource_id: str) -> str:
|
||||
return f"resource://{user_id}/{resource_id}"
|
||||
|
||||
|
||||
def infer_content_type(filename: str | None, mime_type: str | None) -> str:
|
||||
mime = (mime_type or mimetypes.guess_type(filename or "")[0] or "").lower()
|
||||
suffix = Path(filename or "").suffix.lower()
|
||||
if mime.startswith("image/"):
|
||||
return "image"
|
||||
if mime.startswith("audio/"):
|
||||
return "audio"
|
||||
if mime == "application/pdf" or suffix == ".pdf":
|
||||
return "pdf"
|
||||
if mime in {"text/html", "application/xhtml+xml"} or suffix in {".html", ".htm"}:
|
||||
return "html"
|
||||
if mime.startswith("text/plain") or suffix in {".txt", ".md", ".csv", ".log"}:
|
||||
return "text"
|
||||
return "doc"
|
||||
|
||||
|
||||
def _safe_filename(filename: str | None) -> str:
|
||||
name = Path(filename or "upload.bin").name
|
||||
return name or "upload.bin"
|
||||
|
||||
|
||||
def _copy_upload(file: UploadFile, destination: Path) -> tuple[str, int]:
|
||||
sha256 = hashlib.sha256()
|
||||
size = 0
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with destination.open("wb") as out:
|
||||
while True:
|
||||
chunk = file.file.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
sha256.update(chunk)
|
||||
out.write(chunk)
|
||||
return sha256.hexdigest(), size
|
||||
|
||||
|
||||
class MemoryGatewayService:
|
||||
def __init__(
|
||||
self,
|
||||
config: GatewayConfig,
|
||||
repository: MemoryRepository,
|
||||
everos_client: Any,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.repository = repository
|
||||
self.everos_client = everos_client
|
||||
|
||||
def create_user(self, user_id: str) -> dict[str, Any]:
|
||||
user_key = f"uk_{secrets.token_urlsafe(32)}"
|
||||
user = self.repository.create_user(user_id, user_key)
|
||||
return {
|
||||
"user_id": user["id"],
|
||||
"user_key": user["user_key"],
|
||||
"created_at": user["created_at"],
|
||||
}
|
||||
|
||||
def authenticate_user(self, user_id: str, user_key: str) -> bool:
|
||||
user = self.repository.get_user(user_id)
|
||||
if user is None:
|
||||
return False
|
||||
return secrets.compare_digest(str(user["user_key"]), user_key)
|
||||
|
||||
async def upload_resource(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
app_id: str,
|
||||
project_id: str,
|
||||
file: UploadFile,
|
||||
title: str | None,
|
||||
description: str | None,
|
||||
) -> dict[str, Any]:
|
||||
resource_id = new_resource_id()
|
||||
session_id = resource_session_id(user_id, resource_id)
|
||||
original_filename = _safe_filename(file.filename)
|
||||
mime_type = file.content_type or mimetypes.guess_type(original_filename)[0]
|
||||
content_type = infer_content_type(original_filename, mime_type)
|
||||
stored_path = self.config.storage_dir / user_id / resource_id / original_filename
|
||||
sha256, size_bytes = _copy_upload(file, stored_path)
|
||||
internal_uri = stored_path.resolve().as_uri()
|
||||
|
||||
resource = self.repository.create_resource(
|
||||
id=resource_id,
|
||||
user_id=user_id,
|
||||
app_id=app_id,
|
||||
project_id=project_id,
|
||||
session_id=session_id,
|
||||
original_filename=original_filename,
|
||||
mime_type=mime_type,
|
||||
content_type=content_type,
|
||||
uri=internal_uri,
|
||||
uri_public=False,
|
||||
sha256=sha256,
|
||||
size_bytes=size_bytes,
|
||||
title=title,
|
||||
description=description,
|
||||
status="ingesting",
|
||||
error_message=None,
|
||||
)
|
||||
|
||||
try:
|
||||
await self.everos_client.add_memory(
|
||||
self._build_add_payload(
|
||||
resource=resource,
|
||||
user_id=user_id,
|
||||
app_id=app_id,
|
||||
project_id=project_id,
|
||||
filename=original_filename,
|
||||
)
|
||||
)
|
||||
await self.everos_client.flush_memory(session_id, app_id, project_id)
|
||||
except Exception as exc:
|
||||
failed = self.repository.update_resource_status(
|
||||
resource_id,
|
||||
"failed",
|
||||
str(exc),
|
||||
)
|
||||
return self._resource_summary(failed or resource)
|
||||
|
||||
extracted = self.repository.update_resource_status(resource_id, "extracted")
|
||||
return self._resource_summary(extracted or resource)
|
||||
|
||||
def _build_add_payload(
|
||||
self,
|
||||
*,
|
||||
resource: dict[str, Any],
|
||||
user_id: str,
|
||||
app_id: str,
|
||||
project_id: str,
|
||||
filename: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"session_id": resource["session_id"],
|
||||
"app_id": app_id,
|
||||
"project_id": project_id,
|
||||
"messages": [
|
||||
{
|
||||
"sender_id": user_id,
|
||||
"role": "user",
|
||||
"timestamp": 1781068800000,
|
||||
"content": [
|
||||
{
|
||||
"type": resource["content_type"],
|
||||
"uri": resource["uri"],
|
||||
"name": filename,
|
||||
"ext": Path(filename).suffix.lstrip(".") or None,
|
||||
"extras": {
|
||||
"resource_id": resource["id"],
|
||||
"source": "user_upload",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
def list_resources(self, user_id: str) -> list[dict[str, Any]]:
|
||||
return [self._resource_detail(item) for item in self.repository.list_resources(user_id)]
|
||||
|
||||
def get_resource_detail(
|
||||
self,
|
||||
resource_id: str,
|
||||
user_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
resource = self.repository.get_resource_for_user(resource_id, user_id)
|
||||
if resource is None:
|
||||
return None
|
||||
return self._resource_detail(resource)
|
||||
|
||||
def delete_resource(self, resource_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
before = self.repository.get_resource_for_user(resource_id, user_id)
|
||||
if before is None:
|
||||
return None
|
||||
resource = self.repository.soft_delete_resource(resource_id, user_id)
|
||||
return self._resource_summary(resource)
|
||||
|
||||
async def search_memories(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
query: str,
|
||||
conversation_id: str | None,
|
||||
scope: list[str],
|
||||
top_k: int,
|
||||
app_id: str,
|
||||
project_id: str,
|
||||
) -> dict[str, Any]:
|
||||
results: list[dict[str, Any]] = []
|
||||
session_resource_map: dict[str, dict[str, Any]] = {}
|
||||
|
||||
if "current_chat" in scope and conversation_id:
|
||||
payload = self._search_payload(
|
||||
user_id=user_id,
|
||||
query=query,
|
||||
top_k=top_k,
|
||||
app_id=app_id,
|
||||
project_id=project_id,
|
||||
filters={"session_id": f"chat:{conversation_id}"},
|
||||
)
|
||||
results.extend(
|
||||
self._extract_results(
|
||||
await self.everos_client.search_memory(payload),
|
||||
source_scope="current_chat",
|
||||
session_resource_map=session_resource_map,
|
||||
user_id=user_id,
|
||||
)
|
||||
)
|
||||
|
||||
if "resources" in scope:
|
||||
resources = self.repository.list_extracted_resources(
|
||||
user_id,
|
||||
app_id,
|
||||
project_id,
|
||||
)
|
||||
session_resource_map.update({item["session_id"]: item for item in resources})
|
||||
session_ids = [item["session_id"] for item in resources]
|
||||
for batch in _chunks(session_ids, self.config.resource_search_batch_size):
|
||||
payload = self._search_payload(
|
||||
user_id=user_id,
|
||||
query=query,
|
||||
top_k=top_k,
|
||||
app_id=app_id,
|
||||
project_id=project_id,
|
||||
filters={"session_id": {"in": batch}},
|
||||
)
|
||||
results.extend(
|
||||
self._extract_results(
|
||||
await self.everos_client.search_memory(payload),
|
||||
source_scope="resources",
|
||||
session_resource_map=session_resource_map,
|
||||
user_id=user_id,
|
||||
)
|
||||
)
|
||||
|
||||
if "all_user_memory" in scope:
|
||||
payload = self._search_payload(
|
||||
user_id=user_id,
|
||||
query=query,
|
||||
top_k=top_k,
|
||||
app_id=app_id,
|
||||
project_id=project_id,
|
||||
filters=None,
|
||||
)
|
||||
results.extend(
|
||||
self._extract_results(
|
||||
await self.everos_client.search_memory(payload),
|
||||
source_scope="all_user_memory",
|
||||
session_resource_map=session_resource_map,
|
||||
user_id=user_id,
|
||||
)
|
||||
)
|
||||
|
||||
filtered = self._apply_tombstones(user_id, results)
|
||||
overridden = self._apply_overrides(user_id, filtered)
|
||||
return {"results": overridden}
|
||||
|
||||
def _search_payload(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
query: str,
|
||||
top_k: int,
|
||||
app_id: str,
|
||||
project_id: str,
|
||||
filters: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"user_id": user_id,
|
||||
"query": query,
|
||||
"top_k": top_k,
|
||||
"app_id": app_id,
|
||||
"project_id": project_id,
|
||||
}
|
||||
if filters is not None:
|
||||
payload["filters"] = filters
|
||||
return payload
|
||||
|
||||
def _extract_results(
|
||||
self,
|
||||
response: dict[str, Any],
|
||||
*,
|
||||
source_scope: str,
|
||||
session_resource_map: dict[str, dict[str, Any]],
|
||||
user_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
data = response.get("data", {})
|
||||
raw_items: list[dict[str, Any]] = []
|
||||
for key in (
|
||||
"episodes",
|
||||
"profiles",
|
||||
"agent_cases",
|
||||
"agent_skills",
|
||||
"unprocessed_messages",
|
||||
):
|
||||
raw_items.extend(data.get(key, []) or [])
|
||||
|
||||
normalized = []
|
||||
for raw in raw_items:
|
||||
session_id = raw.get("session_id")
|
||||
resource = session_resource_map.get(session_id)
|
||||
if resource is None and isinstance(session_id, str):
|
||||
resource = self.repository.get_resource_by_session_for_user(
|
||||
session_id,
|
||||
user_id,
|
||||
)
|
||||
normalized.append(
|
||||
{
|
||||
"id": raw.get("id"),
|
||||
"session_id": session_id,
|
||||
"text": _display_text(raw),
|
||||
"score": raw.get("score"),
|
||||
"source_scope": source_scope,
|
||||
"resource_id": resource["id"] if resource else None,
|
||||
"resource_uri": (
|
||||
public_resource_uri(user_id, resource["id"]) if resource else None
|
||||
),
|
||||
"raw": raw,
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
def _apply_tombstones(
|
||||
self,
|
||||
user_id: str,
|
||||
results: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
tombstones = self.repository.get_tombstones(user_id)
|
||||
memory_ids = {item["memory_id"] for item in tombstones if item["memory_id"]}
|
||||
session_ids = {item["session_id"] for item in tombstones if item["session_id"]}
|
||||
return [
|
||||
item
|
||||
for item in results
|
||||
if item.get("id") not in memory_ids
|
||||
and item.get("session_id") not in session_ids
|
||||
]
|
||||
|
||||
def _apply_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
results: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
overrides = {
|
||||
item["memory_id"]: item
|
||||
for item in self.repository.get_active_overrides(user_id)
|
||||
if item["memory_id"]
|
||||
}
|
||||
for result in results:
|
||||
override = overrides.get(result.get("id"))
|
||||
if override:
|
||||
result["text"] = override["override_text"]
|
||||
result["override_id"] = override["id"]
|
||||
return results
|
||||
|
||||
def upsert_override(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
memory_id: str,
|
||||
session_id: str | None,
|
||||
override_text: str,
|
||||
) -> dict[str, Any]:
|
||||
override = self.repository.upsert_override(
|
||||
user_id,
|
||||
memory_id,
|
||||
session_id,
|
||||
override_text,
|
||||
)
|
||||
return {"memory_id": memory_id, "override_id": override["id"], "status": "active"}
|
||||
|
||||
def delete_memory(
|
||||
self,
|
||||
*,
|
||||
user_id: str,
|
||||
memory_id: str,
|
||||
session_id: str | None,
|
||||
reason: str | None,
|
||||
) -> dict[str, Any]:
|
||||
tombstone = self.repository.add_tombstone(
|
||||
user_id,
|
||||
memory_id,
|
||||
session_id,
|
||||
reason,
|
||||
)
|
||||
return {"memory_id": memory_id, "tombstone_id": tombstone["id"], "status": "deleted"}
|
||||
|
||||
def _resource_summary(self, resource: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"resource_id": resource["id"],
|
||||
"session_id": resource["session_id"],
|
||||
"uri": public_resource_uri(resource["user_id"], resource["id"]),
|
||||
"status": resource["status"],
|
||||
}
|
||||
|
||||
def _resource_detail(self, resource: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"resource_id": resource["id"],
|
||||
"user_id": resource["user_id"],
|
||||
"filename": resource["original_filename"],
|
||||
"content_type": resource["content_type"],
|
||||
"mime_type": resource["mime_type"],
|
||||
"uri": public_resource_uri(resource["user_id"], resource["id"]),
|
||||
"session_id": resource["session_id"],
|
||||
"status": resource["status"],
|
||||
"title": resource["title"],
|
||||
"description": resource["description"],
|
||||
"created_at": resource["created_at"],
|
||||
"updated_at": resource["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
def _chunks(items: list[str], size: int) -> list[list[str]]:
|
||||
if not items:
|
||||
return []
|
||||
return [items[index : index + size] for index in range(0, len(items), size)]
|
||||
|
||||
|
||||
def _display_text(raw: dict[str, Any]) -> str:
|
||||
for key in (
|
||||
"episode",
|
||||
"summary",
|
||||
"content",
|
||||
"profile_data",
|
||||
"task_intent",
|
||||
"approach",
|
||||
"key_insight",
|
||||
"name",
|
||||
"description",
|
||||
):
|
||||
value = raw.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return str(value)
|
||||
return ""
|
||||
Reference in New Issue
Block a user