Simplify to memory system api

This commit is contained in:
2026-05-18 09:54:26 +08:00
parent b226749c61
commit e689b13e4a
134 changed files with 982 additions and 14575 deletions

View File

@ -0,0 +1 @@
"""Lightweight Memory System API package."""

69
memory_system_api/api.py Normal file
View File

@ -0,0 +1,69 @@
"""FastAPI router for the lightweight Memory System API."""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, Query, status
from .auth import verify_api_key
from .schemas import MessageIngestRequest, SearchRequest, SessionUserRequest
from .service import MemorySystemService
router = APIRouter(
prefix="/memory-system",
tags=["memory-system"],
dependencies=[Depends(verify_api_key)],
)
def get_service() -> MemorySystemService:
return MemorySystemService()
@router.get("/health")
async def health(service: MemorySystemService = Depends(get_service)):
return await service.health()
@router.post("/messages")
async def ingest_messages(request: MessageIngestRequest, service: MemorySystemService = Depends(get_service)):
try:
return await service.ingest_messages(request)
except ValueError as exc:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc
@router.post("/sessions/{session_id}/commit")
async def commit_session(
session_id: str,
request: SessionUserRequest,
service: MemorySystemService = Depends(get_service),
):
return await service.commit_session(request.user_id, session_id)
@router.post("/sessions/{session_id}/extract")
async def extract_session(
session_id: str,
request: SessionUserRequest,
service: MemorySystemService = Depends(get_service),
):
return await service.extract_session(request.user_id, session_id)
@router.get("/openviking/tasks/{task_id}")
async def get_openviking_task(
task_id: str,
user_id: str = Query(min_length=1),
service: MemorySystemService = Depends(get_service),
):
return await service.get_openviking_task(user_id, task_id)
@router.post("/search")
async def search(request: SearchRequest, service: MemorySystemService = Depends(get_service)):
return await service.search(request)
@router.get("/users/{user_id}/profile")
async def get_profile(user_id: str, service: MemorySystemService = Depends(get_service)):
return await service.get_profile(user_id)

12
memory_system_api/auth.py Normal file
View File

@ -0,0 +1,12 @@
"""API key auth for Memory System API."""
from __future__ import annotations
from fastapi import Header, HTTPException, status
from .config import get_config
def verify_api_key(x_api_key: str | None = Header(default=None)) -> None:
expected = get_config().server.api_key
if expected and x_api_key != expected:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API key")

View File

@ -0,0 +1,210 @@
"""Async clients for OpenViking and EverOS used by the lightweight API."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
import httpx
from .config import get_config
from .store import OpenVikingUserKeyStore
class OpenVikingMemorySystemClient:
def __init__(self, store: OpenVikingUserKeyStore | None = None) -> None:
config = get_config()
self.base_url = config.openviking.url.rstrip("/")
self.root_key = config.openviking.api_key or "your-secret-root-key"
self.timeout = config.openviking.timeout
self.verify_ssl = config.openviking.verify_ssl
self.store = store or OpenVikingUserKeyStore(config.storage.sqlite_path)
async def health(self) -> dict[str, Any]:
async with self._client(self.root_key) as client:
response = await client.get("/health")
response.raise_for_status()
return response.json()
async def ensure_user(self, user_id: str) -> str:
existing = self.store.get_user_key(user_id)
if existing:
return existing
async with self._client(self.root_key) as client:
response = await client.post(
"/api/v1/admin/accounts",
json={"account_id": user_id, "admin_user_id": user_id},
)
response.raise_for_status()
data = response.json()
user_key = self._extract_user_key(data)
if not user_key:
raise RuntimeError("OpenViking did not return user_key")
self.store.save_user_key(user_id, user_key)
return user_key
async def ensure_session(self, user_key: str, session_id: str) -> dict[str, Any]:
async with self._client(user_key) as client:
response = await client.post("/api/v1/sessions", json={"session_id": session_id})
if response.status_code in {409, 422}:
return {"session_id": session_id, "status": "exists"}
response.raise_for_status()
return response.json()
async def append_message(self, user_key: str, session_id: str, role: str, content: str) -> dict[str, Any]:
async with self._client(user_key) as client:
response = await client.post(
f"/api/v1/sessions/{session_id}/messages",
json={"role": role, "content": content},
)
response.raise_for_status()
return response.json()
async def commit_session(self, user_key: str, session_id: str) -> dict[str, Any]:
async with self._client(user_key) as client:
response = await client.post(f"/api/v1/sessions/{session_id}/commit")
response.raise_for_status()
return response.json()
async def extract_session(self, user_key: str, session_id: str) -> dict[str, Any]:
async with self._client(user_key) as client:
response = await client.post(f"/api/v1/sessions/{session_id}/extract")
response.raise_for_status()
return response.json()
async def get_task(self, user_key: str, task_id: str) -> dict[str, Any]:
async with self._client(user_key) as client:
response = await client.get(f"/api/v1/tasks/{task_id}")
response.raise_for_status()
return response.json()
async def find(self, user_key: str, user_id: str, query: str, limit: int) -> dict[str, Any]:
async with self._client(user_key) as client:
response = await client.post(
"/api/v1/search/find",
json={
"query": query,
"target_uri": f"viking://user/{user_id}/memories/",
"limit": limit,
},
)
response.raise_for_status()
return response.json()
async def search(self, user_key: str, session_id: str | None, query: str, limit: int) -> dict[str, Any]:
payload: dict[str, Any] = {"query": query, "limit": limit}
if session_id:
payload["session_id"] = session_id
async with self._client(user_key) as client:
response = await client.post("/api/v1/search/search", json=payload)
response.raise_for_status()
return response.json()
def _client(self, api_key: str) -> httpx.AsyncClient:
return httpx.AsyncClient(
base_url=self.base_url,
headers={"X-API-Key": api_key, "Content-Type": "application/json"},
timeout=self.timeout,
verify=self.verify_ssl,
)
def _extract_user_key(self, data: dict[str, Any]) -> str | None:
result = data.get("result") if isinstance(data.get("result"), dict) else data
value = result.get("user_key") if isinstance(result, dict) else None
return str(value) if value else None
class EverOSMemorySystemClient:
def __init__(self) -> None:
config = get_config()
self.base_url = config.everos.url.rstrip("/")
self.api_key = config.everos.api_key
self.timeout = config.everos.timeout
self.verify_ssl = config.everos.verify_ssl
self.health_path = config.everos.health_path
async def health(self) -> dict[str, Any]:
async with self._client() as client:
response = await client.get(self.health_path)
response.raise_for_status()
return response.json()
async def append_message(self, user_id: str, session_id: str, role: str, content: str) -> dict[str, Any]:
async with self._client() as client:
response = await client.post(
"/api/v1/memories",
json=self.build_message_payload(user_id=user_id, session_id=session_id, role=role, content=content),
)
response.raise_for_status()
return response.json()
def build_message_payload(self, user_id: str, session_id: str, role: str, content: str) -> dict[str, Any]:
everos_role = "assistant" if role == "assistant" else "user"
sender_id = "assistant" if everos_role == "assistant" else user_id
timestamp = int(datetime.now(timezone.utc).timestamp() * 1000)
return {
"user_id": user_id,
"session_id": session_id,
"messages": [
{
"message_id": f"msg_{timestamp}",
"timestamp": timestamp,
"sender_id": sender_id,
"sender_name": sender_id,
"role": everos_role,
"content": content,
}
],
}
async def flush(self, user_id: str, session_id: str) -> dict[str, Any]:
async with self._client() as client:
response = await client.post("/api/v1/memories/flush", json={"user_id": user_id, "session_id": session_id})
response.raise_for_status()
return response.json()
async def search(self, user_id: str, session_id: str | None, query: str, method: str, limit: int) -> dict[str, Any]:
filters: dict[str, Any] = {"user_id": user_id}
if session_id:
filters["session_id"] = session_id
async with self._client() as client:
response = await client.post(
"/api/v1/memories/search",
json={
"query": query,
"method": method,
"memory_types": ["episodic_memory", "profile", "raw_message"],
"filters": filters,
"top_k": limit,
"include_original_data": True,
},
)
response.raise_for_status()
return response.json()
async def get_profile(self, user_id: str) -> dict[str, Any]:
async with self._client() as client:
response = await client.post(
"/api/v1/memories/get",
json={
"memory_type": "profile",
"filters": {"user_id": user_id},
"page": 1,
"page_size": 20,
},
)
response.raise_for_status()
return response.json()
def _client(self) -> httpx.AsyncClient:
headers = {"Content-Type": "application/json"}
if self.api_key:
headers["X-API-Key"] = self.api_key
headers["Authorization"] = f"Bearer {self.api_key}"
return httpx.AsyncClient(
base_url=self.base_url,
headers=headers,
timeout=self.timeout,
verify=self.verify_ssl,
)

100
memory_system_api/config.py Normal file
View File

@ -0,0 +1,100 @@
"""Configuration loading for Memory System API."""
from __future__ import annotations
import os
from pathlib import Path
from typing import Any, Literal
import yaml
from pydantic import BaseModel, Field
class ServerConfig(BaseModel):
host: str = "127.0.0.1"
port: int = 1934
api_key: str = ""
class OpenVikingConfig(BaseModel):
url: str = "http://127.0.0.1:1933"
api_key: str = ""
timeout: int = 30
verify_ssl: bool = True
class EverOSConfig(BaseModel):
url: str = "http://127.0.0.1:1995"
api_key: str = ""
timeout: int = 30
verify_ssl: bool = True
health_path: str = "/health"
class StorageConfig(BaseModel):
sqlite_path: str = "/home/tom/memory-gateway/memory_system_api.sqlite3"
class LoggingConfig(BaseModel):
level: str = "INFO"
class Config(BaseModel):
server: ServerConfig = Field(default_factory=ServerConfig)
openviking: OpenVikingConfig = Field(default_factory=OpenVikingConfig)
everos: EverOSConfig = Field(default_factory=EverOSConfig)
storage: StorageConfig = Field(default_factory=StorageConfig)
logging: LoggingConfig = Field(default_factory=LoggingConfig)
_config: Config | None = None
def load_config(config_path: str | None = None) -> Config:
path = Path(config_path or os.environ.get("MEMORY_SYSTEM_CONFIG", "config.yaml"))
if not path.exists():
return _apply_env_overrides(Config())
with path.open("r", encoding="utf-8") as handle:
data = yaml.safe_load(handle) or {}
config = Config(
server=ServerConfig(**data.get("server", {})),
openviking=OpenVikingConfig(**data.get("openviking", {})),
everos=EverOSConfig(**data.get("everos", {})),
storage=StorageConfig(**data.get("storage", {})),
logging=LoggingConfig(**data.get("logging", {})),
)
return _apply_env_overrides(config)
def get_config() -> Config:
global _config
if _config is None:
_config = load_config()
return _config
def set_config(config: Config) -> None:
global _config
_config = config
def _apply_env_overrides(config: Config) -> Config:
updates: dict[str, dict[str, Any]] = {
"server": _env_updates("MEMORY_SYSTEM_SERVER", {"API_KEY": "api_key", "HOST": "host", "PORT": "port"}),
"openviking": _env_updates("OPENVIKING", {"URL": "url", "BASE_URL": "url", "API_KEY": "api_key", "TIMEOUT": "timeout"}),
"everos": _env_updates("EVEROS", {"URL": "url", "BASE_URL": "url", "API_KEY": "api_key", "TIMEOUT": "timeout"}),
"storage": _env_updates("MEMORY_SYSTEM_STORAGE", {"SQLITE_PATH": "sqlite_path"}),
}
for section, values in updates.items():
if values:
setattr(config, section, getattr(config, section).model_copy(update=values))
return config
def _env_updates(prefix: str, mapping: dict[str, str]) -> dict[str, Any]:
values: dict[str, Any] = {}
for env_name, field_name in mapping.items():
raw = os.environ.get(f"{prefix}_{env_name}")
if raw is None:
continue
values[field_name] = int(raw) if field_name in {"port", "timeout"} else raw
return values

View File

@ -0,0 +1,64 @@
"""Schemas for the lightweight Memory System API."""
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
OperationStatus = Literal["success", "partial_success", "failed"]
class MessageIngestRequest(BaseModel):
user_id: str = Field(min_length=1)
session_id: str = Field(min_length=1)
user_message: str | None = None
assistant_message: str | None = None
timestamp: int | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class SessionUserRequest(BaseModel):
user_id: str = Field(min_length=1)
class SearchRequest(BaseModel):
user_id: str = Field(min_length=1)
session_id: str | None = None
query: str = Field(min_length=1)
use_llm: bool = False
limit: int = Field(default=10, ge=1, le=100)
class BackendStatus(BaseModel):
status: OperationStatus
result: Any = None
error: str | None = None
class MessageIngestResponse(BaseModel):
status: OperationStatus
message_count: int
backends: dict[str, BackendStatus]
class CommitResponse(BaseModel):
status: OperationStatus
backends: dict[str, BackendStatus]
class ExtractResponse(BaseModel):
status: OperationStatus
backends: dict[str, BackendStatus]
class SearchResponse(BaseModel):
status: OperationStatus
items: list[dict[str, Any]] = Field(default_factory=list)
backends: dict[str, BackendStatus]
class ProfileResponse(BaseModel):
status: OperationStatus
profile: Any = None
backends: dict[str, BackendStatus]

View File

@ -0,0 +1,49 @@
"""Standalone FastAPI server for Memory System API."""
from __future__ import annotations
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from .api import router
from .config import Config, load_config, set_config
app = FastAPI(title="Memory System API", version="0.1.0")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(router)
def create_app(config: Config | None = None) -> FastAPI:
if config:
set_config(config)
return app
def main() -> None:
import argparse
import uvicorn
parser = argparse.ArgumentParser(description="Memory System API")
parser.add_argument("--config", default="config.yaml", help="Config file path")
parser.add_argument("--host", default=None, help="Bind host")
parser.add_argument("--port", type=int, default=None, help="Bind port")
args = parser.parse_args()
config = load_config(args.config)
if args.host:
config.server.host = args.host
if args.port:
config.server.port = args.port
set_config(config)
uvicorn.run(app, host=config.server.host, port=config.server.port, log_level=config.logging.level.lower())
if __name__ == "__main__":
main()

View File

@ -0,0 +1,165 @@
"""Orchestration for the lightweight Memory System API."""
from __future__ import annotations
import asyncio
from typing import Any, Awaitable, Callable
from .clients import EverOSMemorySystemClient, OpenVikingMemorySystemClient
from .schemas import (
BackendStatus,
CommitResponse,
ExtractResponse,
MessageIngestRequest,
MessageIngestResponse,
ProfileResponse,
SearchRequest,
SearchResponse,
)
class MemorySystemService:
def __init__(self, openviking: Any | None = None, everos: Any | None = None) -> None:
self.openviking = openviking or OpenVikingMemorySystemClient()
self.everos = everos or EverOSMemorySystemClient()
async def ingest_messages(self, request: MessageIngestRequest) -> MessageIngestResponse:
messages = self._messages_from_request(request)
if not messages:
raise ValueError("at least one message is required")
user_key = await self.openviking.ensure_user(request.user_id)
await self.openviking.ensure_session(user_key, request.session_id)
print("user_key:", user_key) # Debugging line to check the user_key value
async def write_openviking() -> list[dict[str, Any]]:
results = []
for message in messages:
results.append(
await self.openviking.append_message(user_key, request.session_id, message["role"], message["content"])
)
return results
async def write_everos() -> list[dict[str, Any]]:
results = []
for message in messages:
results.append(
await self.everos.append_message(request.user_id, request.session_id, message["role"], message["content"])
)
return results
backends = await self._run_backends(openviking=write_openviking, everos=write_everos)
return MessageIngestResponse(
status=self._aggregate_status(backends),
message_count=len(messages),
backends=backends,
)
async def commit_session(self, user_id: str, session_id: str) -> CommitResponse:
user_key = await self.openviking.ensure_user(user_id)
async def commit_openviking() -> dict[str, Any]:
return await self.openviking.commit_session(user_key, session_id)
async def flush_everos() -> dict[str, Any]:
return await self.everos.flush(user_id, session_id)
backends = await self._run_backends(openviking=commit_openviking, everos=flush_everos)
return CommitResponse(status=self._aggregate_status(backends), backends=backends)
async def extract_session(self, user_id: str, session_id: str) -> ExtractResponse:
user_key = await self.openviking.ensure_user(user_id)
backends = {
"openviking": await self._capture(lambda: self.openviking.extract_session(user_key, session_id)),
}
return ExtractResponse(status=self._aggregate_status(backends), backends=backends)
async def get_openviking_task(self, user_id: str, task_id: str) -> dict[str, Any]:
user_key = await self.openviking.ensure_user(user_id)
return await self.openviking.get_task(user_key, task_id)
async def search(self, request: SearchRequest) -> SearchResponse:
user_key = await self.openviking.ensure_user(request.user_id)
everos_method = "agentic" if request.use_llm else "hybrid"
async def search_openviking() -> dict[str, Any]:
if request.use_llm:
return await self.openviking.search(user_key, request.session_id, request.query, request.limit)
return await self.openviking.find(user_key, request.user_id, request.query, request.limit)
async def search_everos() -> dict[str, Any]:
return await self.everos.search(
request.user_id,
request.session_id,
request.query,
everos_method,
request.limit,
)
backends = await self._run_backends(openviking=search_openviking, everos=search_everos)
items = self._merge_search_items(backends)
return SearchResponse(status=self._aggregate_status(backends), items=items[: request.limit], backends=backends)
async def get_profile(self, user_id: str) -> ProfileResponse:
backends = {"everos": await self._capture(lambda: self.everos.get_profile(user_id))}
profile = backends["everos"].result if backends["everos"].status == "success" else None
return ProfileResponse(status=self._aggregate_status(backends), profile=profile, backends=backends)
async def health(self) -> dict[str, Any]:
backends = await self._run_backends(openviking=self.openviking.health, everos=self.everos.health)
return {"status": self._aggregate_status(backends), "backends": backends}
def _messages_from_request(self, request: MessageIngestRequest) -> list[dict[str, str]]:
messages = []
if request.user_message:
messages.append({"role": "user", "content": request.user_message})
if request.assistant_message:
messages.append({"role": "assistant", "content": request.assistant_message})
return messages
async def _run_backends(self, **calls: Callable[[], Awaitable[Any]]) -> dict[str, BackendStatus]:
names = list(calls)
results = await asyncio.gather(*(self._capture(calls[name]) for name in names))
return dict(zip(names, results))
async def _capture(self, call: Callable[[], Awaitable[Any]]) -> BackendStatus:
try:
return BackendStatus(status="success", result=await call())
except Exception as exc: # noqa: BLE001
return BackendStatus(status="failed", error=str(exc))
def _aggregate_status(self, backends: dict[str, BackendStatus]) -> str:
statuses = {backend.status for backend in backends.values()}
if statuses == {"success"}:
return "success"
if "success" in statuses:
return "partial_success"
return "failed"
def _merge_search_items(self, backends: dict[str, BackendStatus]) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for backend_name, backend in backends.items():
if backend.status != "success":
continue
items.extend(self._items_from_backend_result(backend_name, backend.result))
return items
def _items_from_backend_result(self, backend_name: str, result: Any) -> list[dict[str, Any]]:
if isinstance(result, dict) and isinstance(result.get("items"), list):
return [self._with_backend(backend_name, item) for item in result["items"] if isinstance(item, dict)]
data = result.get("data") if isinstance(result, dict) and isinstance(result.get("data"), dict) else result
if not isinstance(data, dict):
return []
if isinstance(data.get("result"), dict):
data = data["result"]
raw_items: list[dict[str, Any]] = []
for key in ("memories", "resources", "episodes", "profiles", "raw_messages"):
values = data.get(key)
if isinstance(values, list):
raw_items.extend(item for item in values if isinstance(item, dict))
return [self._with_backend(backend_name, item) for item in raw_items]
def _with_backend(self, backend_name: str, item: dict[str, Any]) -> dict[str, Any]:
if "source_backend" in item:
return item
return {"source_backend": backend_name, **item}

View File

@ -0,0 +1,53 @@
"""Small SQLite store for OpenViking user keys."""
from __future__ import annotations
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
class OpenVikingUserKeyStore:
def __init__(self, sqlite_path: str) -> None:
self.sqlite_path = sqlite_path
self._ensure_table()
def get_user_key(self, user_id: str) -> str | None:
with self._connect() as conn:
row = conn.execute(
"SELECT user_key FROM memory_system_openviking_users WHERE user_id = ?",
(user_id,),
).fetchone()
return str(row[0]) if row else None
def save_user_key(self, user_id: str, user_key: str) -> None:
now = datetime.now(timezone.utc).isoformat()
with self._connect() as conn:
conn.execute(
"""
INSERT INTO memory_system_openviking_users (user_id, account_id, user_key, created_at, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(user_id) DO UPDATE SET
user_key = excluded.user_key,
updated_at = excluded.updated_at
""",
(user_id, user_id, user_key, now, now),
)
def _ensure_table(self) -> None:
path = Path(self.sqlite_path)
path.parent.mkdir(parents=True, exist_ok=True)
with self._connect() as conn:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS memory_system_openviking_users (
user_id TEXT PRIMARY KEY,
account_id TEXT NOT NULL,
user_key TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
)
"""
)
def _connect(self) -> sqlite3.Connection:
return sqlite3.connect(self.sqlite_path)