tomtan/memory-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor code structure for improved readability and maintainability

Browse Source
This commit is contained in:
tomtan
2026-05-08 17:40:11 +08:00
parent 602c2bd71b
commit 0acee1ec6c
20 changed files with 5410 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
memory_gateway/api_v2.py Normal file
View File

@ -0,0 +1,90 @@
"""Memory Gateway v2 workflow API."""
from __future__ import annotations
from typing import Optional
from fastapi import APIRouter, Depends, Query
from .schemas_v2 import (
BackendRefStatus,
BackendType,
CommitJobView,
CommitRequest,
CommitResponse,
FeedbackRequest,
FeedbackResponse,
IngestRequest,
IngestResponse,
MemoryRefType,
MemoryRefView,
OutboxProcessResponse,
RetrieveRequest,
RetrieveResponse,
)
from .server_auth import verify_api_key_compat
from .services_v2 import v2_service
router = APIRouter(prefix="/v2", tags=["memory-v2"], dependencies=[Depends(verify_api_key_compat)])
@router.post("/conversations/ingest", response_model=IngestResponse)
async def ingest_conversation(request: IngestRequest):
return await v2_service.ingest_conversation_turn(request)
@router.post("/conversations/{session_id}/commit", response_model=CommitResponse)
async def commit_conversation(session_id: str, request: CommitRequest):
return await v2_service.commit_session(session_id, request)
@router.get("/jobs/{job_id}", response_model=CommitJobView)
async def get_commit_job(job_id: str):
return v2_service.get_commit_job_view(job_id)
@router.post("/context/retrieve", response_model=RetrieveResponse)
async def retrieve_context(request: RetrieveRequest):
return await v2_service.retrieve_context(request)
@router.get("/memory/refs", response_model=list[MemoryRefView])
async def list_memory_refs(
workspace_id: Optional[str] = Query(default=None),
user_id: Optional[str] = Query(default=None),
agent_id: Optional[str] = Query(default=None),
session_id: Optional[str] = Query(default=None),
namespace: Optional[str] = Query(default=None),
backend_type: Optional[BackendType] = Query(default=None),
ref_type: Optional[MemoryRefType] = Query(default=None),
status: Optional[BackendRefStatus] = Query(default=None),
limit: int = Query(default=100, ge=1, le=1000),
):
return v2_service.list_memory_refs(
workspace_id=workspace_id,
user_id=user_id,
agent_id=agent_id,
session_id=session_id,
namespace=namespace,
backend_type=backend_type,
ref_type=ref_type,
status=status,
limit=limit,
)
@router.post("/memory/feedback", response_model=FeedbackResponse)
async def memory_feedback(request: FeedbackRequest):
return await v2_service.record_memory_feedback(request)
@router.post("/admin/outbox/process", response_model=OutboxProcessResponse, tags=["memory-v2-admin"])
async def process_outbox(
limit: int = Query(default=100, ge=1, le=1000),
worker_id: Optional[str] = Query(default=None),
lease_seconds: int = Query(default=300, ge=1, le=3600),
):
return await v2_service.process_pending_outbox_events_summary(
limit=limit,
worker_id=worker_id,
lease_seconds=lease_seconds,
)

129
memory_gateway/backend_adapter_mapping.py Normal file
View File

@ -0,0 +1,129 @@
"""Contract-first mapping spec for future v2 backend adapters.
This module intentionally does not call OpenViking, EverMemOS, or Obsidian.
It documents the stable Gateway control-plane fields that may be persisted in
outbox payload refs, SQLite metadata_json, audit summaries, and related control
records. It is not a validator for transient runtime adapter request objects:
the Gateway may pass conversation content to a backend during the current
request lifecycle, but must not persist that content in SQLite/outbox/audit
control-plane records.
"""
from __future__ import annotations
from typing import Final, NamedTuple
from .backend_contracts import BackendCommitResult, BackendOperation, BackendRetrieveResult, BackendWriteResult
from .schemas_v2 import BackendType
CONTROL_PLANE_PERSISTED_PAYLOAD_FIELDS: Final[frozenset[str]] = frozenset(
{
"event_id",
"gateway_id",
"workspace_id",
"user_id",
"agent_id",
"session_id",
"turn_id",
"namespace",
"source_type",
"source_event_id",
"backend_type",
"operation",
"payload_ref",
"metadata",
"trace",
"role",
}
)
CONTROL_PLANE_PAYLOAD_FIELDS: Final[frozenset[str]] = CONTROL_PLANE_PERSISTED_PAYLOAD_FIELDS
DISALLOWED_PAYLOAD_FIELDS: Final[frozenset[str]] = frozenset(
{
"content",
"raw_request",
"messages",
"conversation",
"transcript",
}
)
class AdapterMappingSpec(NamedTuple):
backend_type: BackendType
operation: BackendOperation
adapter_method: str
backend_capability: str
result_model: type[BackendWriteResult] | type[BackendCommitResult] | type[BackendRetrieveResult]
allowed_payload_fields: frozenset[str] = CONTROL_PLANE_PERSISTED_PAYLOAD_FIELDS
ADAPTER_MAPPING_SPECS: Final[tuple[AdapterMappingSpec, ...]] = (
AdapterMappingSpec(
backend_type=BackendType.OPENVIKING,
operation=BackendOperation.INGEST_TURN,
adapter_method="ingest_conversation_turn",
backend_capability="session archive append / resource context organization",
result_model=BackendWriteResult,
),
AdapterMappingSpec(
backend_type=BackendType.OPENVIKING,
operation=BackendOperation.COMMIT_SESSION,
adapter_method="commit_session_v2",
backend_capability="session commit and session archive ref creation",
result_model=BackendCommitResult,
),
AdapterMappingSpec(
backend_type=BackendType.OPENVIKING,
operation=BackendOperation.RETRIEVE_CONTEXT,
adapter_method="retrieve_context_v2",
backend_capability="runtime session/resource context retrieval",
result_model=BackendRetrieveResult,
),
AdapterMappingSpec(
backend_type=BackendType.EVERMEMOS,
operation=BackendOperation.INGEST_TURN,
adapter_method="ingest_message",
backend_capability="message-level memory ingestion",
result_model=BackendWriteResult,
),
AdapterMappingSpec(
backend_type=BackendType.EVERMEMOS,
operation=BackendOperation.COMMIT_SESSION,
adapter_method="extract_profile_long_term_v2",
backend_capability="episodic/profile/long-term extraction",
result_model=BackendCommitResult,
),
AdapterMappingSpec(
backend_type=BackendType.EVERMEMOS,
operation=BackendOperation.RETRIEVE_CONTEXT,
adapter_method="retrieve_context_v2",
backend_capability="episodic/profile/long-term memory retrieval",
result_model=BackendRetrieveResult,
),
AdapterMappingSpec(
backend_type=BackendType.OBSIDIAN,
operation=BackendOperation.CREATE_REVIEW_DRAFT,
adapter_method="create_review_draft_v2",
backend_capability="human review draft creation for high-risk/high-conflict candidates",
result_model=BackendWriteResult,
),
)
def get_adapter_mapping_spec(backend_type: BackendType, operation: BackendOperation) -> AdapterMappingSpec:
for spec in ADAPTER_MAPPING_SPECS:
if spec.backend_type == backend_type and spec.operation == operation:
return spec
raise KeyError(f"No v2 adapter mapping for {backend_type.value}:{operation.value}")
def validate_control_plane_payload(payload: dict[str, object]) -> None:
"""Validate only persisted control-plane payloads, not runtime adapter requests."""
blocked = sorted(DISALLOWED_PAYLOAD_FIELDS.intersection(payload))
if blocked:
raise ValueError(f"Control-plane persisted payload includes disallowed content fields: {', '.join(blocked)}")
def validate_control_plane_persisted_payload(payload: dict[str, object]) -> None:
validate_control_plane_payload(payload)

134
memory_gateway/backend_contracts.py Normal file
View File

@ -0,0 +1,134 @@
"""Backend adapter contracts for Memory Gateway v2."""
from __future__ import annotations
from datetime import datetime
from enum import Enum
from typing import Any, Optional
from uuid import uuid4
from pydantic import BaseModel, Field
from .schemas import utc_now
from .schemas_v2 import BackendType, MemoryRefType, OperationStatus
class BackendOperation(str, Enum):
INGEST_TURN = "ingest_turn"
COMMIT_SESSION = "commit_session"
RETRIEVE_CONTEXT = "retrieve_context"
CREATE_REVIEW_DRAFT = "create_review_draft"
class BackendResultStatus(str, Enum):
SUCCESS = "success"
FAILED = "failed"
SKIPPED = "skipped"
PENDING = "pending"
class OutboxEventStatus(str, Enum):
PENDING = "pending"
PROCESSING = "processing"
SUCCESS = "success"
SKIPPED = "skipped"
FAILED = "failed"
DEAD_LETTER = "dead_letter"
class BackendWriteResult(BaseModel):
backend_type: BackendType
operation: BackendOperation
status: BackendResultStatus
native_id: Optional[str] = None
native_uri: Optional[str] = None
retryable: bool = False
error_code: Optional[str] = None
error_message: Optional[str] = None
latency_ms: Optional[float] = None
metadata: dict[str, Any] = Field(default_factory=dict)
class BackendProducedRef(BaseModel):
ref_type: MemoryRefType
native_id: Optional[str] = None
native_uri: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)
class BackendCommitResult(BaseModel):
backend_type: BackendType
operation: BackendOperation = BackendOperation.COMMIT_SESSION
status: BackendResultStatus
native_id: Optional[str] = None
native_uri: Optional[str] = None
retryable: bool = False
error_code: Optional[str] = None
error_message: Optional[str] = None
latency_ms: Optional[float] = None
created_refs: list[str] = Field(default_factory=list)
refs: list[BackendProducedRef] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
class BackendRetrieveItem(BaseModel):
text: Optional[str] = None
source_backend: BackendType
ref_id: Optional[str] = None
score: float = 0.0
memory_type: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)
class BackendRetrieveResult(BaseModel):
backend_type: BackendType
operation: BackendOperation = BackendOperation.RETRIEVE_CONTEXT
status: BackendResultStatus
native_id: Optional[str] = None
native_uri: Optional[str] = None
retryable: bool = False
error_code: Optional[str] = None
error_message: Optional[str] = None
latency_ms: Optional[float] = None
items: list[BackendRetrieveItem] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
class OutboxEvent(BaseModel):
id: str = Field(default_factory=lambda: f"outbox_{uuid4().hex[:16]}")
event_type: str
gateway_id: str
workspace_id: str
user_id: str
agent_id: Optional[str] = None
session_id: Optional[str] = None
backend_type: BackendType
operation: BackendOperation
payload_ref: Optional[str] = None
status: OutboxEventStatus = OutboxEventStatus.PENDING
attempt_count: int = 0
max_attempts: int = 3
next_retry_at: Optional[datetime] = None
last_error: Optional[str] = None
locked_by: Optional[str] = None
locked_at: Optional[datetime] = None
lease_expires_at: Optional[datetime] = None
metadata: dict[str, Any] = Field(default_factory=dict)
created_at: datetime = Field(default_factory=utc_now)
updated_at: datetime = Field(default_factory=utc_now)
class CommitJob(BaseModel):
job_id: str = Field(default_factory=lambda: f"job_{uuid4().hex[:16]}")
workspace_id: str
user_id: str
agent_id: Optional[str] = None
session_id: str
namespace: Optional[str] = None
status: OperationStatus = OperationStatus.ACCEPTED
requested_by: Optional[str] = None
created_refs_count: int = 0
error_message: Optional[str] = None
created_at: datetime = Field(default_factory=utc_now)
updated_at: datetime = Field(default_factory=utc_now)
started_at: Optional[datetime] = None
finished_at: Optional[datetime] = None

259
memory_gateway/backend_normalization.py Normal file
View File

@ -0,0 +1,259 @@
"""Offline response normalization helpers for future v2 backend adapters."""
from __future__ import annotations
from typing import Any
from .backend_contracts import (
BackendCommitResult,
BackendOperation,
BackendProducedRef,
BackendResultStatus,
BackendRetrieveItem,
BackendRetrieveResult,
BackendWriteResult,
)
from .backend_ref_mapping import map_backend_ref_type
from .schemas_v2 import BackendType
SAFE_METADATA_KEYS = {
"backend_request_id",
"request_id",
"trace_id",
"latency_ms",
"schema_version",
"source_channel",
"reason",
"original_ref_type",
"confidence",
"score",
"version",
"created_at",
}
BLOCKED_METADATA_KEYS = {"content", "raw_request", "messages", "conversation", "transcript"}
def safe_backend_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
if not metadata:
return {}
safe: dict[str, Any] = {}
for key, value in metadata.items():
if key in BLOCKED_METADATA_KEYS or key not in SAFE_METADATA_KEYS:
continue
if isinstance(value, (str, int, float, bool)) or value is None:
safe[key] = value
return safe
def normalize_openviking_commit_response(raw: dict[str, Any]) -> BackendCommitResult:
status = _result_status(raw)
refs = [_produced_ref(BackendType.OPENVIKING, item) for item in _extract_ref_items(raw)]
return BackendCommitResult(
backend_type=BackendType.OPENVIKING,
operation=BackendOperation.COMMIT_SESSION,
status=status,
native_id=raw.get("native_id") or raw.get("session_id"),
native_uri=raw.get("native_uri") or raw.get("uri"),
retryable=_retryable_from_raw(BackendType.OPENVIKING, raw),
error_code=raw.get("error_code"),
error_message=raw.get("error") or raw.get("error_message"),
latency_ms=raw.get("latency_ms"),
refs=refs,
metadata=safe_backend_metadata(raw.get("metadata") or raw),
)
def normalize_evermemos_commit_response(raw: dict[str, Any]) -> BackendCommitResult:
status = _result_status(raw)
refs = [_produced_ref(BackendType.EVERMEMOS, item) for item in _extract_ref_items(raw)]
return BackendCommitResult(
backend_type=BackendType.EVERMEMOS,
operation=BackendOperation.COMMIT_SESSION,
status=status,
native_id=raw.get("native_id") or raw.get("session_id"),
native_uri=raw.get("native_uri") or raw.get("uri"),
retryable=_retryable_from_raw(BackendType.EVERMEMOS, raw),
error_code=raw.get("error_code"),
error_message=raw.get("error") or raw.get("error_message"),
latency_ms=raw.get("latency_ms"),
refs=refs,
metadata=safe_backend_metadata(raw.get("metadata") or raw),
)
def normalize_openviking_ingest_response(raw: dict[str, Any]) -> BackendWriteResult:
return _write_result(BackendType.OPENVIKING, raw)
def normalize_evermemos_ingest_response(raw: dict[str, Any]) -> BackendWriteResult:
return _write_result(BackendType.EVERMEMOS, raw)
def normalize_openviking_retrieve_response(raw: dict[str, Any]) -> BackendRetrieveResult:
return _retrieve_result(BackendType.OPENVIKING, raw)
def normalize_evermemos_retrieve_response(raw: dict[str, Any]) -> BackendRetrieveResult:
return _retrieve_result(BackendType.EVERMEMOS, raw)
def map_backend_error_to_retryable(
backend_type: BackendType,
status_code: int | None = None,
error_code: str | None = None,
error_message: str | None = None,
) -> bool:
"""Map backend errors into retryable/non-retryable categories.
Unknown errors default to retryable because adapter contracts are still
unstable and transient backend/API rollout failures are more likely during
integration.
"""
if status_code in {429, 500, 502, 503, 504}:
return True
if status_code in {400, 401, 403, 404, 422}:
return False
text = f"{error_code or ''} {error_message or ''}".lower()
if "timeout" in text or "network_error" in text or "connection" in text:
return True
if "validation" in text or "unauthorized" in text or "forbidden" in text or "not_found" in text:
return False
return True
def _write_result(backend_type: BackendType, raw: dict[str, Any]) -> BackendWriteResult:
data = raw.get("result") or raw.get("data") or {}
if not isinstance(data, dict):
data = {}
native_id = (
raw.get("native_id")
or raw.get("id")
or raw.get("memory_id")
or raw.get("session_id")
or data.get("native_id")
or data.get("id")
or data.get("memory_id")
or data.get("session_id")
)
native_uri = (
raw.get("native_uri")
or raw.get("uri")
or raw.get("url")
or data.get("native_uri")
or data.get("uri")
or data.get("url")
)
if not native_uri and backend_type == BackendType.OPENVIKING and native_id:
native_uri = f"viking://sessions/{native_id}"
return BackendWriteResult(
backend_type=backend_type,
operation=BackendOperation.INGEST_TURN,
status=_result_status(raw),
native_id=native_id,
native_uri=native_uri,
retryable=_retryable_from_raw(backend_type, raw),
error_code=raw.get("error_code"),
error_message=raw.get("error") or raw.get("error_message"),
latency_ms=raw.get("latency_ms"),
metadata=safe_backend_metadata(raw.get("metadata") or raw),
)
def _retrieve_result(backend_type: BackendType, raw: dict[str, Any]) -> BackendRetrieveResult:
if not isinstance(raw, dict) or not raw:
return BackendRetrieveResult(
backend_type=backend_type,
operation=BackendOperation.RETRIEVE_CONTEXT,
status=BackendResultStatus.SKIPPED,
metadata={"reason": "malformed_or_empty_response"},
)
return BackendRetrieveResult(
backend_type=backend_type,
operation=BackendOperation.RETRIEVE_CONTEXT,
status=_result_status(raw),
native_id=raw.get("native_id") or raw.get("session_id"),
native_uri=raw.get("native_uri") or raw.get("uri"),
retryable=_retryable_from_raw(backend_type, raw),
error_code=raw.get("error_code"),
error_message=raw.get("error") or raw.get("error_message"),
latency_ms=raw.get("latency_ms"),
items=[_retrieve_item(backend_type, item) for item in _extract_retrieve_items(raw)],
metadata=safe_backend_metadata(raw.get("metadata") or raw),
)
def _retrieve_item(backend_type: BackendType, item: dict[str, Any]) -> BackendRetrieveItem:
metadata = safe_backend_metadata(item.get("metadata") if isinstance(item.get("metadata"), dict) else item)
return BackendRetrieveItem(
text=item.get("text") or item.get("summary") or item.get("abstract"),
source_backend=backend_type,
ref_id=item.get("ref_id") or item.get("id") or item.get("memory_id") or item.get("profile_id") or item.get("session_id") or item.get("uri"),
score=float(item.get("score") or 0.0),
memory_type=item.get("memory_type") or item.get("ref_type") or item.get("type") or item.get("kind"),
metadata=metadata,
)
def _produced_ref(backend_type: BackendType, item: dict[str, Any]) -> BackendProducedRef:
ref_type, mapping_metadata = map_backend_ref_type(backend_type, item.get("ref_type") or item.get("type") or item.get("kind"))
metadata = {
**safe_backend_metadata(item.get("metadata") if isinstance(item.get("metadata"), dict) else item),
**mapping_metadata,
}
return BackendProducedRef(
ref_type=ref_type,
native_id=item.get("native_id") or item.get("id") or item.get("memory_id") or item.get("profile_id") or item.get("session_id"),
native_uri=item.get("native_uri") or item.get("uri") or item.get("url"),
metadata=metadata,
)
def _extract_ref_items(raw: dict[str, Any]) -> list[dict[str, Any]]:
data = raw.get("result") or raw.get("data") or raw
candidates = (
data.get("refs")
or data.get("produced_refs")
or data.get("created_refs")
or data.get("memories")
or data.get("items")
or []
)
return [item for item in candidates if isinstance(item, dict)]
def _extract_retrieve_items(raw: dict[str, Any]) -> list[dict[str, Any]]:
data = raw.get("result") or raw.get("data") or raw
if not isinstance(data, dict):
return []
candidates = (
data.get("items")
or data.get("results")
or data.get("memories")
or data.get("resources")
or data.get("contexts")
or []
)
return [item for item in candidates if isinstance(item, dict)]
def _result_status(raw: dict[str, Any]) -> BackendResultStatus:
status = str(raw.get("status") or "success").lower()
if status in {"ok", "created", "accepted"}:
return BackendResultStatus.SUCCESS
try:
return BackendResultStatus(status)
except ValueError:
return BackendResultStatus.SUCCESS if not raw.get("error") and not raw.get("error_message") else BackendResultStatus.FAILED
def _retryable_from_raw(backend_type: BackendType, raw: dict[str, Any]) -> bool:
if "retryable" in raw:
return bool(raw["retryable"])
if raw.get("error") or raw.get("error_message") or raw.get("error_code") or raw.get("status_code"):
return map_backend_error_to_retryable(
backend_type,
status_code=raw.get("status_code"),
error_code=raw.get("error_code"),
error_message=raw.get("error") or raw.get("error_message"),
)
return False

66
memory_gateway/backend_ref_mapping.py Normal file
View File

@ -0,0 +1,66 @@
"""Backend-specific ref type mapping for Memory Gateway v2."""
from __future__ import annotations
from .schemas_v2 import BackendType, MemoryRefType
OPENVIKING_REF_TYPE_MAP = {
"session_archive": MemoryRefType.SESSION_ARCHIVE,
"context_resource": MemoryRefType.CONTEXT_RESOURCE,
"resource": MemoryRefType.CONTEXT_RESOURCE,
"session_summary": MemoryRefType.SESSION_ARCHIVE,
}
EVERMEMOS_REF_TYPE_MAP = {
"message_memory": MemoryRefType.MESSAGE_MEMORY,
"episodic_memory": MemoryRefType.EPISODIC_MEMORY,
"episode": MemoryRefType.EPISODIC_MEMORY,
"profile": MemoryRefType.PROFILE,
"long_term_memory": MemoryRefType.LONG_TERM_MEMORY,
"memory": MemoryRefType.LONG_TERM_MEMORY,
"preference": MemoryRefType.PROFILE,
}
OBSIDIAN_REF_TYPE_MAP = {
"review_draft": MemoryRefType.DRAFT_REVIEW,
"draft_review": MemoryRefType.DRAFT_REVIEW,
}
def map_backend_ref_type(
backend_type: BackendType,
backend_ref_type: str | None,
) -> tuple[MemoryRefType, dict[str, str]]:
"""Map backend-native ref type to a Gateway MemoryRefType.
Unknown values fall back to a backend-appropriate default and preserve the
original value in returned metadata for later inspection.
"""
raw_type = (backend_ref_type or "").strip()
normalized = raw_type.lower()
if backend_type == BackendType.OPENVIKING:
mapped = OPENVIKING_REF_TYPE_MAP.get(normalized, MemoryRefType.SESSION_ARCHIVE)
elif backend_type == BackendType.EVERMEMOS:
mapped = EVERMEMOS_REF_TYPE_MAP.get(normalized, MemoryRefType.LONG_TERM_MEMORY)
elif backend_type == BackendType.OBSIDIAN:
mapped = OBSIDIAN_REF_TYPE_MAP.get(normalized, MemoryRefType.DRAFT_REVIEW)
else:
mapped = MemoryRefType.LONG_TERM_MEMORY
metadata: dict[str, str] = {}
if raw_type and raw_type not in {mapped.value, normalized}:
metadata["original_ref_type"] = raw_type
elif raw_type and normalized not in _known_backend_ref_types(backend_type):
metadata["original_ref_type"] = raw_type
return mapped, metadata
def _known_backend_ref_types(backend_type: BackendType) -> set[str]:
if backend_type == BackendType.OPENVIKING:
return set(OPENVIKING_REF_TYPE_MAP)
if backend_type == BackendType.EVERMEMOS:
return set(EVERMEMOS_REF_TYPE_MAP)
if backend_type == BackendType.OBSIDIAN:
return set(OBSIDIAN_REF_TYPE_MAP)
return set()

48
memory_gateway/config.py
View File

@ -18,16 +18,16 @@ def load_config(config_path: Optional[str] = None) -> Config:
if not config_file.exists():
# 返回默认配置
return Config()
return _apply_env_overrides(Config())
try:
with open(config_file, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
if data is None:
return Config()
return _apply_env_overrides(Config())
return Config(
config = Config(
server=ServerConfig(**data.get("server", {})),
openviking=OpenVikingConfig(**data.get("openviking", {})),
evermemos=EverMemOSConfig(**data.get("evermemos", {})),
@ -37,9 +37,10 @@ def load_config(config_path: Optional[str] = None) -> Config:
obsidian=ObsidianConfig(**data.get("obsidian", {})),
storage=StorageConfig(**data.get("storage", {})),
)
return _apply_env_overrides(config)
except (ValidationError, yaml.YAMLError) as e:
print(f"配置文件解析错误: {e}")
return Config()
return _apply_env_overrides(Config())
def get_config() -> Config:
@ -57,3 +58,42 @@ def set_config(config: Config) -> None:
_config: Optional[Config] = None
def _apply_env_overrides(config: Config) -> Config:
openviking_updates = _backend_env_updates("OPENVIKING")
evermemos_updates = _backend_env_updates("EVERMEMOS")
if openviking_updates:
config.openviking = config.openviking.model_copy(update=openviking_updates)
if evermemos_updates:
config.evermemos = config.evermemos.model_copy(update=evermemos_updates)
return config
def _backend_env_updates(prefix: str) -> dict:
updates = {}
env_map = {
"ENABLED": "enabled",
"MODE": "mode",
"BASE_URL": "url",
"URL": "url",
"API_KEY": "api_key",
"TOKEN": "api_key",
"TIMEOUT": "timeout",
"TIMEOUT_SECONDS": "timeout",
"VERIFY_SSL": "verify_ssl",
"INGEST_PATH": "ingest_path",
}
for env_name, field_name in env_map.items():
value = os.environ.get(f"{prefix}_{env_name}")
if value is None:
continue
if field_name == "enabled":
updates[field_name] = value.lower() in {"1", "true", "yes", "on"}
elif field_name == "timeout":
updates[field_name] = int(value)
elif field_name == "verify_ssl":
updates[field_name] = value.lower() not in {"0", "false", "no", "off"}
else:
updates[field_name] = value
return updates

206
memory_gateway/evermemos_client.py
View File

@ -1,12 +1,21 @@
"""Client for the external EverMemOS consolidation service."""
from __future__ import annotations
from json import JSONDecodeError
from typing import Any
import httpx
from .backend_contracts import BackendCommitResult, BackendOperation, BackendResultStatus, BackendRetrieveResult, BackendWriteResult
from .backend_normalization import (
map_backend_error_to_retryable,
normalize_evermemos_commit_response,
normalize_evermemos_ingest_response,
normalize_evermemos_retrieve_response,
)
from .config import get_config
from .schemas import AccessContext, EpisodeRecord, MemoryRecord
from .schemas_v2 import BackendType
class EverMemOSError(RuntimeError):
@ -26,15 +35,25 @@ class EverMemOSClient:
base_url: str | None = None,
api_key: str | None = None,
timeout: int | None = None,
enabled: bool | None = None,
mode: str | None = None,
verify_ssl: bool | None = None,
health_path: str | None = None,
ingest_path: str | None = None,
consolidate_path: str | None = None,
transport: httpx.BaseTransport | None = None,
) -> None:
config = get_config().evermemos
self.base_url = (base_url or config.url).rstrip("/")
self.base_url = (base_url if base_url is not None else config.url).rstrip("/")
self.api_key = api_key if api_key is not None else config.api_key
self.timeout = timeout or config.timeout
self.enabled = config.enabled if enabled is None else enabled
self.mode = mode or config.mode
self.verify_ssl = config.verify_ssl if verify_ssl is None else verify_ssl
self.health_path = health_path or config.health_path
self.ingest_path = ingest_path or config.ingest_path
self.consolidate_path = consolidate_path or config.consolidate_path
self.transport = transport
def _headers(self) -> dict[str, str]:
headers = {"Content-Type": "application/json"}
@ -46,13 +65,195 @@ class EverMemOSClient:
def health(self) -> dict[str, Any]:
url = self.base_url + self.health_path
try:
with httpx.Client(timeout=self.timeout, headers=self._headers()) as client:
health_timeout = httpx.Timeout(min(self.timeout, 2.0), connect=min(self.timeout, 0.5))
with httpx.Client(timeout=health_timeout, headers=self._headers()) as client:
response = client.get(url)
response.raise_for_status()
return {"status": "ok", "url": self.base_url, "response": response.json()}
except Exception as exc: # noqa: BLE001
return {"status": "error", "url": self.base_url, "error": str(exc)}
def ingest_message(self, payload: dict[str, Any]) -> BackendWriteResult:
"""v2 adapter placeholder for message-level EverMemOS ingestion.
Mapping spec: `backend_adapter_mapping.AdapterMappingSpec` maps
EverMemOS ingest_turn to this method and requires BackendWriteResult.
Payloads must contain only control-plane fields; raw request bodies are
not persisted by the Gateway control-plane store.
TODO(v2): bind this to EverMemOS `/api/v1/memories` or its stable
message ingestion API after the external contract settles.
"""
runtime_payload = self._build_ingest_payload(payload)
if self._use_real_api:
return self._ingest_message_real(runtime_payload)
raw = {
"status": "skipped",
"memory_id": runtime_payload.get("turn_id"),
"metadata": {
"reason": "evermemos_v2_ingest_adapter_not_configured",
"schema_version": "evermemos.fixture.ingest.v2",
},
}
return self._normalize_ingest_response(raw)
@property
def _use_real_api(self) -> bool:
# Real ingest is strictly gated by mode=real. The legacy `enabled`
# field is retained for config compatibility, but must not trigger
# network traffic by itself.
return self.mode == "real"
def _ingest_message_real(self, runtime_payload: dict[str, Any]) -> BackendWriteResult:
if not self.base_url:
return self._failed_ingest_result(
error_code="config_error",
error_message="EverMemOS real ingest is enabled but base_url is missing",
retryable=False,
)
try:
with httpx.Client(
base_url=self.base_url,
headers=self._headers(),
timeout=self.timeout,
verify=self.verify_ssl,
transport=self.transport,
) as client:
response = client.post(self.ingest_path, json=runtime_payload)
if response.status_code >= 400:
return self._failed_ingest_result(
error_code=f"http_{response.status_code}",
error_message=f"EverMemOS ingest failed with HTTP {response.status_code}",
retryable=self._map_error(response),
)
try:
raw = response.json()
except (JSONDecodeError, ValueError):
return self._failed_ingest_result(
error_code="invalid_json",
error_message="EverMemOS ingest returned invalid JSON",
retryable=True,
)
if not isinstance(raw, dict):
return self._failed_ingest_result(
error_code="unexpected_response",
error_message="EverMemOS ingest returned an unexpected response shape",
retryable=True,
)
return self._normalize_ingest_response(raw)
except httpx.TimeoutException as exc:
return self._failed_ingest_result("timeout", self._safe_error_message(exc), retryable=self._map_error(exc))
except httpx.RequestError as exc:
return self._failed_ingest_result("network_error", self._safe_error_message(exc), retryable=self._map_error(exc))
except Exception as exc: # noqa: BLE001
return self._failed_ingest_result("unexpected_error", self._safe_error_message(exc), retryable=self._map_error(exc))
def extract_profile_long_term_v2(self, payload: dict[str, Any]) -> BackendCommitResult:
"""v2 adapter placeholder for profile / long-term extraction.
Mapping spec: commit_session returns BackendCommitResult and should
produce native episodic/profile/long-term refs once the real API is stable.
"""
runtime_payload = self._build_commit_payload(payload)
raw = {
"status": "success",
"session_id": runtime_payload.get("session_id"),
"metadata": {
"reason": "evermemos_v2_commit_fixture",
"schema_version": "evermemos.fixture.commit.v2",
},
"data": {
"produced_refs": [
{
"ref_type": "profile",
"profile_id": f"em_profile:{runtime_payload.get('user_id') or 'unknown'}",
"metadata": {"schema_version": "evermemos.fixture.profile.v2"},
},
{
"ref_type": "long_term_memory",
"memory_id": f"em_long_term:{runtime_payload.get('session_id')}",
"metadata": {"schema_version": "evermemos.fixture.long_term.v2"},
},
]
},
}
return self._normalize_commit_response(raw)
def retrieve_context_v2(self, payload: dict[str, Any]) -> BackendRetrieveResult:
"""v2 adapter placeholder for episodic/profile/long-term retrieval.
Mapping spec: retrieve_context returns BackendRetrieveResult with
normalized context items, not raw backend payload dumps.
"""
raw = {
"status": "success",
"metadata": {
"reason": "evermemos_v2_retrieve_fixture",
"schema_version": "evermemos.fixture.retrieve.v2",
},
"data": {
"items": [
{
"text": "EverMemOS fixture profile context.",
"profile_id": f"em_profile:{payload.get('user_id') or 'unknown'}",
"score": 0.72,
"memory_type": "profile",
"metadata": {"schema_version": "evermemos.fixture.retrieve.item.v2"},
},
{
"text": "EverMemOS fixture long-term memory context.",
"memory_id": f"em_long_term:{payload.get('session_id') or 'unknown'}",
"score": 0.69,
"memory_type": "long_term_memory",
"metadata": {"schema_version": "evermemos.fixture.retrieve.item.v2"},
},
]
},
}
return self._normalize_retrieve_response(raw)
def _build_ingest_payload(self, payload: dict[str, Any]) -> dict[str, Any]:
# Runtime-only adapter payload. It may include conversation content for
# the current request lifecycle; callers must not persist it to SQLite.
return dict(payload)
def _build_commit_payload(self, payload: dict[str, Any]) -> dict[str, Any]:
return dict(payload)
def _normalize_ingest_response(self, raw: dict[str, Any]) -> BackendWriteResult:
return normalize_evermemos_ingest_response(raw)
def _normalize_commit_response(self, raw: dict[str, Any]) -> BackendCommitResult:
return normalize_evermemos_commit_response(raw)
def _normalize_retrieve_response(self, raw: dict[str, Any]) -> BackendRetrieveResult:
return normalize_evermemos_retrieve_response(raw)
def _map_error(self, exc_or_response: Any) -> bool:
status_code = getattr(exc_or_response, "status_code", None)
error_code = getattr(exc_or_response, "error_code", None)
error_message = str(exc_or_response) if exc_or_response is not None else None
return map_backend_error_to_retryable(
BackendType.EVERMEMOS,
status_code=status_code,
error_code=error_code,
error_message=error_message,
)
def _failed_ingest_result(self, error_code: str, error_message: str, retryable: bool) -> BackendWriteResult:
return BackendWriteResult(
backend_type=BackendType.EVERMEMOS,
operation=BackendOperation.INGEST_TURN,
status=BackendResultStatus.FAILED,
retryable=retryable,
error_code=error_code,
error_message=error_message,
metadata={"error_code": error_code},
)
def _safe_error_message(self, exc: Exception) -> str:
return exc.__class__.__name__
def consolidate_session(
self,
session_id: str,
@ -110,4 +311,3 @@ class EverMemOSClient:
"conflicts": data.get("conflicts") or [],
"review_drafts": data.get("review_drafts") or [],
}

43
memory_gateway/evermemos_service.py
View File

@ -13,6 +13,7 @@ conflicting candidates.
from __future__ import annotations
import argparse
import hashlib
import logging
from typing import Any
@ -37,6 +38,18 @@ class ConsolidateRequest(BaseModel):
existing_memories: list[dict[str, Any]] = Field(default_factory=list)
class MemoryIngestRequest(BaseModel):
workspace_id: str | None = None
user_id: str
session_id: str
turn_id: str
role: str = "user"
content: str
metadata: dict[str, Any] = Field(default_factory=dict)
source_type: str | None = None
source_event_id: str | None = None
app = FastAPI(title="Local EverMemOS POC Service", version="0.1.0")
@ -50,6 +63,35 @@ async def health() -> dict[str, Any]:
}
@app.post("/api/v1/memories")
async def ingest_memory(request: MemoryIngestRequest) -> dict[str, Any]:
"""Accept message-level ingest for local real-adapter smoke tests.
This POC endpoint intentionally does not persist raw conversation content.
It only returns a stable backend reference that Memory Gateway can store as
control-plane metadata.
"""
seed = "|".join(
[
request.workspace_id or "",
request.user_id,
request.session_id,
request.turn_id,
request.source_event_id or "",
]
)
memory_id = "em_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]
return {
"status": "success",
"memory_id": memory_id,
"native_uri": f"evermemos://memories/{memory_id}",
"metadata": {
"schema_version": "evermemos.local.ingest.v1",
"source_channel": request.metadata.get("source_channel") or request.metadata.get("channel"),
},
}
@app.post("/v1/sessions/consolidate")
async def consolidate_session(request: ConsolidateRequest) -> dict[str, Any]:
repo = InMemoryRepository()
@ -105,4 +147,3 @@ def main() -> None:
if __name__ == "__main__":
main()

25
memory_gateway/obsidian_review_client.py Normal file
View File

@ -0,0 +1,25 @@
"""Human-review backend skeleton for Memory Gateway v2.
Obsidian remains a human-in-the-loop review backend only. This skeleton does
not write files or call external APIs; it preserves the adapter contract until
the review draft integration is explicitly designed.
"""
from __future__ import annotations
from typing import Any
from .backend_contracts import BackendOperation, BackendResultStatus, BackendWriteResult
from .schemas_v2 import BackendType
class ObsidianReviewClient:
def create_review_draft_v2(self, payload: dict[str, Any]) -> BackendWriteResult:
"""Return a skipped review-draft result until the real adapter exists."""
return BackendWriteResult(
backend_type=BackendType.OBSIDIAN,
operation=BackendOperation.CREATE_REVIEW_DRAFT,
status=BackendResultStatus.SKIPPED,
native_id=payload.get("event_id") or payload.get("gateway_id"),
retryable=False,
metadata={"reason": "obsidian_review_adapter_not_configured"},
)

203
memory_gateway/openviking_client.py
View File

@ -5,12 +5,21 @@ import json
import logging
import mimetypes
import tempfile
from json import JSONDecodeError
from pathlib import Path
from typing import Any, Optional
import httpx
from .backend_contracts import BackendCommitResult, BackendOperation, BackendResultStatus, BackendRetrieveResult, BackendWriteResult
from .backend_normalization import (
map_backend_error_to_retryable,
normalize_openviking_commit_response,
normalize_openviking_ingest_response,
normalize_openviking_retrieve_response,
)
from .config import get_config
from .schemas_v2 import BackendType
from .types import MemoryEntry, ResourceEntry, SearchResult
logger = logging.getLogger(__name__)
@ -23,16 +32,26 @@ class OpenVikingClient:
self,
base_url: Optional[str] = None,
api_key: Optional[str] = None,
timeout: int = 30,
timeout: int | None = None,
account: str = "default",
user: str = "default",
enabled: bool | None = None,
mode: str | None = None,
verify_ssl: bool | None = None,
ingest_path: str | None = None,
transport: httpx.AsyncBaseTransport | None = None,
):
self.config = get_config()
self.base_url = base_url or self.config.openviking.url
self.api_key = api_key or self.config.openviking.api_key or "your-secret-root-key"
self.timeout = timeout
self.base_url = base_url if base_url is not None else self.config.openviking.url
self.api_key = api_key if api_key is not None else (self.config.openviking.api_key or "your-secret-root-key")
self.timeout = timeout if timeout is not None else self.config.openviking.timeout
self.account = account
self.user = user
self.enabled = self.config.openviking.enabled if enabled is None else enabled
self.mode = mode or self.config.openviking.mode
self.verify_ssl = self.config.openviking.verify_ssl if verify_ssl is None else verify_ssl
self.ingest_path = ingest_path or self.config.openviking.ingest_path
self.transport = transport
self._client: Optional[httpx.AsyncClient] = None
def _get_headers(self) -> dict[str, str]:
@ -49,6 +68,8 @@ class OpenVikingClient:
base_url=self.base_url,
headers=self._get_headers(),
timeout=self.timeout,
verify=self.verify_ssl,
transport=self.transport,
)
return self._client
@ -67,6 +88,180 @@ class OpenVikingClient:
logger.error(f"OpenViking 健康检查失败: {e}")
return {"status": "error", "message": str(e)}
async def ingest_conversation_turn(self, payload: dict[str, Any]) -> BackendWriteResult:
"""v2 adapter placeholder for OpenViking session archive ingestion.
Mapping spec: `backend_adapter_mapping.AdapterMappingSpec` maps
OpenViking ingest_turn to this method and requires BackendWriteResult.
Payloads must contain only control-plane fields; conversation content
is not persisted by the Gateway control-plane store.
TODO(v2): bind this to OpenViking's stable session/message archive API
once that contract is finalized. Until then the gateway records a
skipped backend ref instead of inventing an unstable HTTP contract.
"""
runtime_payload = self._build_ingest_payload(payload)
if self._use_real_api:
return await self._ingest_conversation_turn_real(runtime_payload)
raw = {
"status": "skipped",
"session_id": runtime_payload.get("session_id"),
"uri": f"viking://sessions/{runtime_payload.get('session_id')}",
"metadata": {
"reason": "openviking_v2_ingest_adapter_not_configured",
"schema_version": "openviking.fixture.ingest.v2",
},
}
return self._normalize_ingest_response(raw)
@property
def _use_real_api(self) -> bool:
# Real ingest is strictly gated by mode=real. The legacy `enabled`
# field is retained for config compatibility, but must not trigger
# network traffic by itself.
return self.mode == "real"
async def _ingest_conversation_turn_real(self, runtime_payload: dict[str, Any]) -> BackendWriteResult:
if not self.base_url:
return self._failed_ingest_result(
error_code="config_error",
error_message="OpenViking real ingest is enabled but base_url is missing",
retryable=False,
)
try:
client = await self._get_client()
response = await client.post(
self._format_ingest_path(runtime_payload),
json=runtime_payload,
)
if response.status_code >= 400:
return self._failed_ingest_result(
error_code=f"http_{response.status_code}",
error_message=f"OpenViking ingest failed with HTTP {response.status_code}",
retryable=self._map_error(response),
)
try:
raw = response.json()
except (JSONDecodeError, ValueError):
return self._failed_ingest_result(
error_code="invalid_json",
error_message="OpenViking ingest returned invalid JSON",
retryable=True,
)
if not isinstance(raw, dict):
return self._failed_ingest_result(
error_code="unexpected_response",
error_message="OpenViking ingest returned an unexpected response shape",
retryable=True,
)
return self._normalize_ingest_response(raw)
except httpx.TimeoutException as exc:
return self._failed_ingest_result("timeout", self._safe_error_message(exc), retryable=self._map_error(exc))
except httpx.RequestError as exc:
return self._failed_ingest_result("network_error", self._safe_error_message(exc), retryable=self._map_error(exc))
except Exception as exc: # noqa: BLE001
return self._failed_ingest_result("unexpected_error", self._safe_error_message(exc), retryable=self._map_error(exc))
async def commit_session_v2(self, payload: dict[str, Any]) -> BackendCommitResult:
"""v2 adapter placeholder for OpenViking session commit.
Mapping spec: commit_session returns BackendCommitResult and should
produce a native session/archive ref once the real API is stable.
"""
runtime_payload = self._build_commit_payload(payload)
raw = {
"status": "success",
"session_id": runtime_payload.get("session_id"),
"metadata": {
"reason": "openviking_v2_commit_fixture",
"schema_version": "openviking.fixture.commit.v2",
},
"result": {
"refs": [
{
"type": "session_summary",
"id": f"ov_session_summary:{runtime_payload.get('session_id')}",
"uri": f"viking://sessions/{runtime_payload.get('session_id')}/summary",
"metadata": {"schema_version": "openviking.fixture.ref.v2"},
}
]
},
}
return self._normalize_commit_response(raw)
async def retrieve_context_v2(self, payload: dict[str, Any]) -> BackendRetrieveResult:
"""v2 adapter placeholder for OpenViking runtime context retrieval.
Mapping spec: retrieve_context returns BackendRetrieveResult with
runtime context items, not raw backend payload dumps.
"""
raw = {
"status": "ok",
"session_id": payload.get("session_id"),
"metadata": {
"reason": "openviking_v2_retrieve_fixture",
"schema_version": "openviking.fixture.retrieve.v2",
},
"result": {
"items": [
{
"text": "OpenViking fixture runtime context.",
"ref_id": f"ov_context:{payload.get('session_id') or 'unknown'}",
"score": 0.75,
"memory_type": "context_resource",
"metadata": {"schema_version": "openviking.fixture.retrieve.item.v2"},
}
]
},
}
return self._normalize_retrieve_response(raw)
def _build_ingest_payload(self, payload: dict[str, Any]) -> dict[str, Any]:
# Runtime-only adapter payload. It may include conversation content for
# the current request lifecycle; callers must not persist it to SQLite.
return dict(payload)
def _format_ingest_path(self, payload: dict[str, Any]) -> str:
session_id = str(payload.get("session_id") or "unknown")
return self.ingest_path.format(session_id=session_id)
def _build_commit_payload(self, payload: dict[str, Any]) -> dict[str, Any]:
return dict(payload)
def _normalize_ingest_response(self, raw: dict[str, Any]) -> BackendWriteResult:
return normalize_openviking_ingest_response(raw)
def _normalize_commit_response(self, raw: dict[str, Any]) -> BackendCommitResult:
return normalize_openviking_commit_response(raw)
def _normalize_retrieve_response(self, raw: dict[str, Any]) -> BackendRetrieveResult:
return normalize_openviking_retrieve_response(raw)
def _map_error(self, exc_or_response: Any) -> bool:
status_code = getattr(exc_or_response, "status_code", None)
error_code = getattr(exc_or_response, "error_code", None)
error_message = str(exc_or_response) if exc_or_response is not None else None
return map_backend_error_to_retryable(
BackendType.OPENVIKING,
status_code=status_code,
error_code=error_code,
error_message=error_message,
)
def _failed_ingest_result(self, error_code: str, error_message: str, retryable: bool) -> BackendWriteResult:
return BackendWriteResult(
backend_type=BackendType.OPENVIKING,
operation=BackendOperation.INGEST_TURN,
status=BackendResultStatus.FAILED,
retryable=retryable,
error_code=error_code,
error_message=error_message,
metadata={"error_code": error_code},
)
def _safe_error_message(self, exc: Exception) -> str:
return exc.__class__.__name__
async def search(
self,
query: str,

779
memory_gateway/repositories.py
View File

@ -7,12 +7,14 @@ from __future__ import annotations
import json
import sqlite3
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Iterable, Optional, Protocol
from .backend_contracts import BackendOperation, CommitJob, OutboxEvent, OutboxEventStatus
from .config import get_config
from .schemas import AuditLog, EpisodeRecord, MemoryRecord, ProfileRecord, UserRecord
from .schemas_v2 import BackendRefStatus, BackendType, MemoryRef, MemoryRefType
class MetadataRepository(Protocol):
@ -28,6 +30,61 @@ class MetadataRepository(Protocol):
def upsert_profile(self, profile: ProfileRecord) -> ProfileRecord: ...
def add_audit(self, audit: AuditLog) -> AuditLog: ...
def list_audit(self, limit: int = 100) -> list[AuditLog]: ...
def save_memory_ref(self, ref: MemoryRef) -> MemoryRef: ...
def get_memory_ref(self, ref_id: str) -> MemoryRef | None: ...
def list_memory_refs(
self,
gateway_id: str | None = None,
workspace_id: str | None = None,
user_id: str | None = None,
agent_id: str | None = None,
session_id: str | None = None,
namespace: str | None = None,
backend_type: BackendType | str | None = None,
ref_type: MemoryRefType | str | None = None,
status: BackendRefStatus | str | None = None,
limit: int = 100,
) -> list[MemoryRef]: ...
def save_outbox_event(self, event: OutboxEvent) -> OutboxEvent: ...
def list_outbox_events(
self,
status: OutboxEventStatus | str | None = None,
backend_type: BackendType | str | None = None,
operation: BackendOperation | str | None = None,
gateway_id: str | None = None,
payload_ref: str | None = None,
limit: int = 100,
) -> list[OutboxEvent]: ...
def list_outbox_events_by_job(self, job_id: str, limit: int = 100) -> list[OutboxEvent]: ...
def claim_outbox_event(self, event_id: str) -> OutboxEvent | None: ...
def claim_pending_outbox_events(
self,
limit: int,
worker_id: str,
lease_seconds: int,
) -> list[OutboxEvent]: ...
def release_expired_processing_events(self, now: datetime | None = None) -> list[OutboxEvent]: ...
def update_outbox_event_status(
self,
event_id: str,
status: OutboxEventStatus | str,
last_error: str | None = None,
) -> OutboxEvent | None: ...
def save_commit_job(self, job: CommitJob) -> CommitJob: ...
def get_commit_job(self, job_id: str) -> CommitJob | None: ...
def update_commit_job_status(
self,
job_id: str,
status: str,
error_message: str | None = None,
created_refs_count: int | None = None,
) -> CommitJob | None: ...
def count_memory_refs(
self,
gateway_id: str | None = None,
session_id: str | None = None,
status: BackendRefStatus | str | None = None,
) -> int: ...
def _json_dump_model(model) -> str:
@ -38,6 +95,14 @@ def _json_load_model(model_cls, payload: str):
return model_cls.model_validate(json.loads(payload))
def _enum_value(value):
return value.value if hasattr(value, "value") else value
def _safe_timedelta(seconds: int) -> timedelta:
return timedelta(seconds=max(1, int(seconds)))
class InMemoryRepository:
def __init__(self) -> None:
self.users: dict[str, UserRecord] = {}
@ -45,6 +110,9 @@ class InMemoryRepository:
self.episodes: dict[str, EpisodeRecord] = {}
self.profiles: dict[str, ProfileRecord] = {}
self.audit_logs: list[AuditLog] = []
self.memory_refs: dict[str, MemoryRef] = {}
self.outbox_events: dict[str, OutboxEvent] = {}
self.commit_jobs: dict[str, CommitJob] = {}
def create_user(self, user: UserRecord) -> UserRecord:
now = datetime.now(timezone.utc)
@ -102,6 +170,210 @@ class InMemoryRepository:
def list_audit(self, limit: int = 100) -> list[AuditLog]:
return self.audit_logs[-limit:]
def save_memory_ref(self, ref: MemoryRef) -> MemoryRef:
now = datetime.now(timezone.utc)
existing = self.memory_refs.get(ref.id)
if existing:
ref.created_at = existing.created_at
ref.updated_at = now
self.memory_refs[ref.id] = ref
return ref
def get_memory_ref(self, ref_id: str) -> MemoryRef | None:
return self.memory_refs.get(ref_id)
def list_memory_refs(
self,
gateway_id: str | None = None,
workspace_id: str | None = None,
user_id: str | None = None,
agent_id: str | None = None,
session_id: str | None = None,
namespace: str | None = None,
backend_type: BackendType | str | None = None,
ref_type: MemoryRefType | str | None = None,
status: BackendRefStatus | str | None = None,
limit: int = 100,
) -> list[MemoryRef]:
refs = list(self.memory_refs.values())
def matches(ref: MemoryRef) -> bool:
return (
(workspace_id is None or ref.workspace_id == workspace_id)
and (gateway_id is None or ref.gateway_id == gateway_id)
and (user_id is None or ref.user_id == user_id)
and (agent_id is None or ref.agent_id == agent_id)
and (session_id is None or ref.session_id == session_id)
and (namespace is None or ref.namespace == namespace)
and (backend_type is None or ref.backend_type.value == _enum_value(backend_type))
and (ref_type is None or ref.ref_type.value == _enum_value(ref_type))
and (status is None or ref.status.value == _enum_value(status))
)
refs = [ref for ref in refs if matches(ref)]
refs.sort(key=lambda ref: ref.created_at, reverse=True)
return refs[:limit]
def save_outbox_event(self, event: OutboxEvent) -> OutboxEvent:
now = datetime.now(timezone.utc)
existing = self.outbox_events.get(event.id)
if existing:
event.created_at = existing.created_at
event.updated_at = now
self.outbox_events[event.id] = event
return event
def list_outbox_events(
self,
status: OutboxEventStatus | str | None = None,
backend_type: BackendType | str | None = None,
operation: BackendOperation | str | None = None,
gateway_id: str | None = None,
payload_ref: str | None = None,
limit: int = 100,
) -> list[OutboxEvent]:
events = list(self.outbox_events.values())
def matches(event: OutboxEvent) -> bool:
return (
(status is None or event.status.value == _enum_value(status))
and (backend_type is None or event.backend_type.value == _enum_value(backend_type))
and (operation is None or event.operation.value == _enum_value(operation))
and (gateway_id is None or event.gateway_id == gateway_id)
and (payload_ref is None or event.payload_ref == payload_ref)
)
events = [event for event in events if matches(event)]
events.sort(key=lambda event: event.created_at, reverse=True)
return events[:limit]
def list_outbox_events_by_job(self, job_id: str, limit: int = 100) -> list[OutboxEvent]:
return self.list_outbox_events(payload_ref=f"commit_job:{job_id}", limit=limit)
def claim_outbox_event(self, event_id: str) -> OutboxEvent | None:
event = self.outbox_events.get(event_id)
now = datetime.now(timezone.utc)
if not event or event.status != OutboxEventStatus.PENDING:
return None
if event.next_retry_at and event.next_retry_at > now:
return None
event.status = OutboxEventStatus.PROCESSING
event.locked_by = "inline"
event.locked_at = now
event.lease_expires_at = now + _safe_timedelta(300)
event.updated_at = now
self.outbox_events[event.id] = event
return event
def claim_pending_outbox_events(
self,
limit: int,
worker_id: str,
lease_seconds: int,
) -> list[OutboxEvent]:
now = datetime.now(timezone.utc)
candidates = [
event
for event in self.outbox_events.values()
if event.status == OutboxEventStatus.PENDING
and (event.next_retry_at is None or event.next_retry_at <= now)
]
candidates.sort(key=lambda event: event.created_at)
claimed: list[OutboxEvent] = []
for event in candidates[:limit]:
event.status = OutboxEventStatus.PROCESSING
event.locked_by = worker_id
event.locked_at = now
event.lease_expires_at = now + _safe_timedelta(lease_seconds)
event.updated_at = now
self.outbox_events[event.id] = event
claimed.append(event)
return claimed
def release_expired_processing_events(self, now: datetime | None = None) -> list[OutboxEvent]:
now = now or datetime.now(timezone.utc)
released: list[OutboxEvent] = []
for event in list(self.outbox_events.values()):
if (
event.status == OutboxEventStatus.PROCESSING
and event.lease_expires_at is not None
and event.lease_expires_at <= now
):
event.status = OutboxEventStatus.PENDING
event.locked_by = None
event.locked_at = None
event.lease_expires_at = None
event.updated_at = now
self.outbox_events[event.id] = event
released.append(event)
return released
def update_outbox_event_status(
self,
event_id: str,
status: OutboxEventStatus | str,
last_error: str | None = None,
) -> OutboxEvent | None:
event = self.outbox_events.get(event_id)
if not event:
return None
event.status = OutboxEventStatus(_enum_value(status))
event.last_error = last_error
event.updated_at = datetime.now(timezone.utc)
if event.status != OutboxEventStatus.PROCESSING:
event.locked_by = None
event.locked_at = None
event.lease_expires_at = None
if event.status in {OutboxEventStatus.FAILED, OutboxEventStatus.DEAD_LETTER}:
event.attempt_count += 1
self.outbox_events[event.id] = event
return event
def save_commit_job(self, job: CommitJob) -> CommitJob:
now = datetime.now(timezone.utc)
existing = self.commit_jobs.get(job.job_id)
if existing:
job.created_at = existing.created_at
job.updated_at = now
self.commit_jobs[job.job_id] = job
return job
def count_memory_refs(
self,
gateway_id: str | None = None,
session_id: str | None = None,
status: BackendRefStatus | str | None = None,
) -> int:
return len(self.list_memory_refs(gateway_id=gateway_id, session_id=session_id, status=status, limit=100000))
def get_commit_job(self, job_id: str) -> CommitJob | None:
return self.commit_jobs.get(job_id)
def update_commit_job_status(
self,
job_id: str,
status: str,
error_message: str | None = None,
created_refs_count: int | None = None,
) -> CommitJob | None:
job = self.commit_jobs.get(job_id)
if not job:
return None
from .schemas_v2 import OperationStatus
job.status = OperationStatus(_enum_value(status))
job.error_message = error_message
if created_refs_count is not None:
job.created_refs_count = created_refs_count
now = datetime.now(timezone.utc)
job.updated_at = now
if job.status.value == "running" and job.started_at is None:
job.started_at = now
if job.status.value in {"success", "failed", "partial_success"}:
job.finished_at = now
self.commit_jobs[job.job_id] = job
return job
class SQLiteRepository:
def __init__(self, db_path: str | Path) -> None:
@ -171,8 +443,121 @@ class SQLiteRepository:
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_audit_created ON audit_logs(created_at);
CREATE TABLE IF NOT EXISTS memory_refs (
id TEXT PRIMARY KEY,
gateway_id TEXT NOT NULL,
workspace_id TEXT NOT NULL,
user_id TEXT NOT NULL,
agent_id TEXT,
session_id TEXT,
turn_id TEXT,
namespace TEXT,
backend_type TEXT NOT NULL,
ref_type TEXT NOT NULL,
native_id TEXT,
native_uri TEXT,
provenance_id TEXT,
idempotency_key TEXT,
content_hash TEXT,
source_type TEXT,
source_event_id TEXT,
status TEXT NOT NULL,
error_message TEXT,
metadata_json TEXT NOT NULL,
payload TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_memory_refs_gateway ON memory_refs(gateway_id);
CREATE INDEX IF NOT EXISTS idx_memory_refs_scope ON memory_refs(workspace_id, user_id, agent_id, session_id);
CREATE INDEX IF NOT EXISTS idx_memory_refs_backend ON memory_refs(backend_type, ref_type, status);
CREATE INDEX IF NOT EXISTS idx_memory_refs_namespace ON memory_refs(namespace);
CREATE TABLE IF NOT EXISTS outbox_events (
id TEXT PRIMARY KEY,
event_type TEXT NOT NULL,
gateway_id TEXT NOT NULL,
workspace_id TEXT NOT NULL,
user_id TEXT NOT NULL,
agent_id TEXT,
session_id TEXT,
backend_type TEXT NOT NULL,
operation TEXT NOT NULL,
payload_ref TEXT,
status TEXT NOT NULL,
attempt_count INTEGER NOT NULL,
max_attempts INTEGER NOT NULL,
next_retry_at TEXT,
last_error TEXT,
locked_by TEXT,
locked_at TEXT,
lease_expires_at TEXT,
metadata_json TEXT NOT NULL,
payload TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_outbox_events_status ON outbox_events(status, next_retry_at);
CREATE INDEX IF NOT EXISTS idx_outbox_events_backend ON outbox_events(backend_type, operation);
CREATE INDEX IF NOT EXISTS idx_outbox_events_gateway ON outbox_events(gateway_id);
CREATE TABLE IF NOT EXISTS commit_jobs (
job_id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL,
user_id TEXT NOT NULL,
agent_id TEXT,
session_id TEXT NOT NULL,
namespace TEXT,
status TEXT NOT NULL,
requested_by TEXT,
created_refs_count INTEGER NOT NULL,
error_message TEXT,
payload TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
started_at TEXT,
finished_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_commit_jobs_session ON commit_jobs(session_id);
CREATE INDEX IF NOT EXISTS idx_commit_jobs_status ON commit_jobs(status);
"""
)
self._ensure_memory_ref_columns(conn)
self._ensure_outbox_event_columns(conn)
conn.execute(
"""
DELETE FROM memory_refs
WHERE rowid NOT IN (
SELECT MAX(rowid)
FROM memory_refs
GROUP BY gateway_id, backend_type, ref_type
)
"""
)
conn.execute(
"""
DROP INDEX IF EXISTS uq_memory_refs_gateway_backend_ref_type
"""
)
def _ensure_memory_ref_columns(self, conn: sqlite3.Connection) -> None:
columns = {row["name"] for row in conn.execute("PRAGMA table_info(memory_refs)").fetchall()}
additions = {
"idempotency_key": "TEXT",
"content_hash": "TEXT",
}
for column, column_type in additions.items():
if column not in columns:
conn.execute(f"ALTER TABLE memory_refs ADD COLUMN {column} {column_type}")
def _ensure_outbox_event_columns(self, conn: sqlite3.Connection) -> None:
columns = {row["name"] for row in conn.execute("PRAGMA table_info(outbox_events)").fetchall()}
additions = {
"locked_by": "TEXT",
"locked_at": "TEXT",
"lease_expires_at": "TEXT",
}
for column, column_type in additions.items():
if column not in columns:
conn.execute(f"ALTER TABLE outbox_events ADD COLUMN {column} {column_type}")
def create_user(self, user: UserRecord) -> UserRecord:
now = datetime.now(timezone.utc)
@ -316,6 +701,397 @@ class SQLiteRepository:
).fetchall()
return [_json_load_model(AuditLog, row["payload"]) for row in rows]
def save_memory_ref(self, ref: MemoryRef) -> MemoryRef:
existing = None
with self._connect() as conn:
row = conn.execute("SELECT payload FROM memory_refs WHERE id = ?", (ref.id,)).fetchone()
if row:
existing = _json_load_model(MemoryRef, row["payload"])
now = datetime.now(timezone.utc)
if existing:
ref.created_at = existing.created_at
ref.updated_at = now
with self._connect() as conn:
conn.execute(
"""
INSERT OR REPLACE INTO memory_refs(
id, gateway_id, workspace_id, user_id, agent_id, session_id, turn_id,
namespace, backend_type, ref_type, native_id, native_uri, provenance_id,
idempotency_key, content_hash, source_type, source_event_id, status, error_message, metadata_json,
payload, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
ref.id,
ref.gateway_id,
ref.workspace_id,
ref.user_id,
ref.agent_id,
ref.session_id,
ref.turn_id,
ref.namespace,
ref.backend_type.value,
ref.ref_type.value,
ref.native_id,
ref.native_uri,
ref.provenance_id,
ref.idempotency_key,
ref.content_hash,
ref.source_type,
ref.source_event_id,
ref.status.value,
ref.error_message,
json.dumps(ref.metadata, ensure_ascii=False),
_json_dump_model(ref),
ref.created_at.isoformat(),
ref.updated_at.isoformat(),
),
)
return ref
def get_memory_ref(self, ref_id: str) -> MemoryRef | None:
with self._connect() as conn:
row = conn.execute("SELECT payload FROM memory_refs WHERE id = ?", (ref_id,)).fetchone()
return _json_load_model(MemoryRef, row["payload"]) if row else None
def list_memory_refs(
self,
gateway_id: str | None = None,
workspace_id: str | None = None,
user_id: str | None = None,
agent_id: str | None = None,
session_id: str | None = None,
namespace: str | None = None,
backend_type: BackendType | str | None = None,
ref_type: MemoryRefType | str | None = None,
status: BackendRefStatus | str | None = None,
limit: int = 100,
) -> list[MemoryRef]:
clauses: list[str] = []
params: list[str | int] = []
filters = {
"gateway_id": gateway_id,
"workspace_id": workspace_id,
"user_id": user_id,
"agent_id": agent_id,
"session_id": session_id,
"namespace": namespace,
"backend_type": _enum_value(backend_type) if backend_type is not None else None,
"ref_type": _enum_value(ref_type) if ref_type is not None else None,
"status": _enum_value(status) if status is not None else None,
}
for key, value in filters.items():
if value is not None:
clauses.append(f"{key} = ?")
params.append(value)
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
params.append(limit)
with self._connect() as conn:
rows = conn.execute(
f"SELECT payload FROM memory_refs {where} ORDER BY created_at DESC LIMIT ?",
params,
).fetchall()
return [_json_load_model(MemoryRef, row["payload"]) for row in rows]
def save_outbox_event(self, event: OutboxEvent) -> OutboxEvent:
existing = None
with self._connect() as conn:
row = conn.execute("SELECT payload FROM outbox_events WHERE id = ?", (event.id,)).fetchone()
if row:
existing = _json_load_model(OutboxEvent, row["payload"])
now = datetime.now(timezone.utc)
if existing:
event.created_at = existing.created_at
event.updated_at = now
with self._connect() as conn:
conn.execute(
"""
INSERT OR REPLACE INTO outbox_events(
id, event_type, gateway_id, workspace_id, user_id, agent_id, session_id,
backend_type, operation, payload_ref, status, attempt_count, max_attempts,
next_retry_at, last_error, locked_by, locked_at, lease_expires_at,
metadata_json, payload, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
event.id,
event.event_type,
event.gateway_id,
event.workspace_id,
event.user_id,
event.agent_id,
event.session_id,
event.backend_type.value,
event.operation.value,
event.payload_ref,
event.status.value,
event.attempt_count,
event.max_attempts,
event.next_retry_at.isoformat() if event.next_retry_at else None,
event.last_error,
event.locked_by,
event.locked_at.isoformat() if event.locked_at else None,
event.lease_expires_at.isoformat() if event.lease_expires_at else None,
json.dumps(event.metadata, ensure_ascii=False),
_json_dump_model(event),
event.created_at.isoformat(),
event.updated_at.isoformat(),
),
)
return event
def list_outbox_events(
self,
status: OutboxEventStatus | str | None = None,
backend_type: BackendType | str | None = None,
operation: BackendOperation | str | None = None,
gateway_id: str | None = None,
payload_ref: str | None = None,
limit: int = 100,
) -> list[OutboxEvent]:
clauses: list[str] = []
params: list[str | int] = []
filters = {
"status": _enum_value(status) if status is not None else None,
"backend_type": _enum_value(backend_type) if backend_type is not None else None,
"operation": _enum_value(operation) if operation is not None else None,
"gateway_id": gateway_id,
"payload_ref": payload_ref,
}
for key, value in filters.items():
if value is not None:
clauses.append(f"{key} = ?")
params.append(value)
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
params.append(limit)
with self._connect() as conn:
rows = conn.execute(
f"SELECT payload FROM outbox_events {where} ORDER BY created_at DESC LIMIT ?",
params,
).fetchall()
return [_json_load_model(OutboxEvent, row["payload"]) for row in rows]
def list_outbox_events_by_job(self, job_id: str, limit: int = 100) -> list[OutboxEvent]:
return self.list_outbox_events(payload_ref=f"commit_job:{job_id}", limit=limit)
def claim_outbox_event(self, event_id: str) -> OutboxEvent | None:
with self._connect() as conn:
row = conn.execute("SELECT payload FROM outbox_events WHERE id = ?", (event_id,)).fetchone()
if not row:
return None
event = _json_load_model(OutboxEvent, row["payload"])
now = datetime.now(timezone.utc)
if event.status != OutboxEventStatus.PENDING:
return None
if event.next_retry_at and event.next_retry_at > now:
return None
event.status = OutboxEventStatus.PROCESSING
event.locked_by = "inline"
event.locked_at = now
event.lease_expires_at = now + _safe_timedelta(300)
event.updated_at = now
with self._connect() as conn:
cursor = conn.execute(
"""
UPDATE outbox_events
SET status = ?, locked_by = ?, locked_at = ?, lease_expires_at = ?,
payload = ?, metadata_json = ?, updated_at = ?
WHERE id = ?
AND status = ?
AND (next_retry_at IS NULL OR next_retry_at <= ?)
""",
(
event.status.value,
event.locked_by,
event.locked_at.isoformat() if event.locked_at else None,
event.lease_expires_at.isoformat() if event.lease_expires_at else None,
_json_dump_model(event),
json.dumps(event.metadata, ensure_ascii=False),
event.updated_at.isoformat(),
event.id,
OutboxEventStatus.PENDING.value,
now.isoformat(),
),
)
return event if cursor.rowcount else None
def claim_pending_outbox_events(
self,
limit: int,
worker_id: str,
lease_seconds: int,
) -> list[OutboxEvent]:
now = datetime.now(timezone.utc)
now_iso = now.isoformat()
with self._connect() as conn:
rows = conn.execute(
"""
SELECT payload FROM outbox_events
WHERE status = ?
AND (next_retry_at IS NULL OR next_retry_at <= ?)
ORDER BY created_at ASC
LIMIT ?
""",
(OutboxEventStatus.PENDING.value, now_iso, limit),
).fetchall()
claimed: list[OutboxEvent] = []
with self._connect() as conn:
for row in rows:
event = _json_load_model(OutboxEvent, row["payload"])
if event.status != OutboxEventStatus.PENDING:
continue
event.status = OutboxEventStatus.PROCESSING
event.locked_by = worker_id
event.locked_at = now
event.lease_expires_at = now + _safe_timedelta(lease_seconds)
event.updated_at = now
cursor = conn.execute(
"""
UPDATE outbox_events
SET status = ?, locked_by = ?, locked_at = ?, lease_expires_at = ?,
payload = ?, metadata_json = ?, updated_at = ?
WHERE id = ?
AND status = ?
AND (next_retry_at IS NULL OR next_retry_at <= ?)
""",
(
event.status.value,
event.locked_by,
event.locked_at.isoformat() if event.locked_at else None,
event.lease_expires_at.isoformat() if event.lease_expires_at else None,
_json_dump_model(event),
json.dumps(event.metadata, ensure_ascii=False),
event.updated_at.isoformat(),
event.id,
OutboxEventStatus.PENDING.value,
now_iso,
),
)
if cursor.rowcount:
claimed.append(event)
return claimed
def release_expired_processing_events(self, now: datetime | None = None) -> list[OutboxEvent]:
now = now or datetime.now(timezone.utc)
with self._connect() as conn:
rows = conn.execute(
"""
SELECT payload FROM outbox_events
WHERE status = ?
AND lease_expires_at IS NOT NULL
AND lease_expires_at <= ?
""",
(OutboxEventStatus.PROCESSING.value, now.isoformat()),
).fetchall()
released: list[OutboxEvent] = []
for row in rows:
event = _json_load_model(OutboxEvent, row["payload"])
event.status = OutboxEventStatus.PENDING
event.locked_by = None
event.locked_at = None
event.lease_expires_at = None
event.updated_at = now
released.append(self.save_outbox_event(event))
return released
def update_outbox_event_status(
self,
event_id: str,
status: OutboxEventStatus | str,
last_error: str | None = None,
) -> OutboxEvent | None:
with self._connect() as conn:
row = conn.execute("SELECT payload FROM outbox_events WHERE id = ?", (event_id,)).fetchone()
if not row:
return None
event = _json_load_model(OutboxEvent, row["payload"])
event.status = OutboxEventStatus(_enum_value(status))
event.last_error = last_error
event.updated_at = datetime.now(timezone.utc)
if event.status != OutboxEventStatus.PROCESSING:
event.locked_by = None
event.locked_at = None
event.lease_expires_at = None
if event.status in {OutboxEventStatus.FAILED, OutboxEventStatus.DEAD_LETTER}:
event.attempt_count += 1
return self.save_outbox_event(event)
def save_commit_job(self, job: CommitJob) -> CommitJob:
existing = None
with self._connect() as conn:
row = conn.execute("SELECT payload FROM commit_jobs WHERE job_id = ?", (job.job_id,)).fetchone()
if row:
existing = _json_load_model(CommitJob, row["payload"])
now = datetime.now(timezone.utc)
if existing:
job.created_at = existing.created_at
job.updated_at = now
with self._connect() as conn:
conn.execute(
"""
INSERT OR REPLACE INTO commit_jobs(
job_id, workspace_id, user_id, agent_id, session_id, namespace,
status, requested_by, created_refs_count, error_message, payload,
created_at, updated_at, started_at, finished_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
job.job_id,
job.workspace_id,
job.user_id,
job.agent_id,
job.session_id,
job.namespace,
job.status.value,
job.requested_by,
job.created_refs_count,
job.error_message,
_json_dump_model(job),
job.created_at.isoformat(),
job.updated_at.isoformat(),
job.started_at.isoformat() if job.started_at else None,
job.finished_at.isoformat() if job.finished_at else None,
),
)
return job
def get_commit_job(self, job_id: str) -> CommitJob | None:
with self._connect() as conn:
row = conn.execute("SELECT payload FROM commit_jobs WHERE job_id = ?", (job_id,)).fetchone()
return _json_load_model(CommitJob, row["payload"]) if row else None
def update_commit_job_status(
self,
job_id: str,
status: str,
error_message: str | None = None,
created_refs_count: int | None = None,
) -> CommitJob | None:
from .schemas_v2 import OperationStatus
job = self.get_commit_job(job_id)
if not job:
return None
job.status = OperationStatus(_enum_value(status))
job.error_message = error_message
if created_refs_count is not None:
job.created_refs_count = created_refs_count
now = datetime.now(timezone.utc)
job.updated_at = now
if job.status.value == "running" and job.started_at is None:
job.started_at = now
if job.status.value in {"success", "failed", "partial_success"}:
job.finished_at = now
return self.save_commit_job(job)
def count_memory_refs(
self,
gateway_id: str | None = None,
session_id: str | None = None,
status: BackendRefStatus | str | None = None,
) -> int:
return len(self.list_memory_refs(gateway_id=gateway_id, session_id=session_id, status=status, limit=100000))
def build_repository() -> MetadataRepository:
config = get_config()
@ -325,4 +1101,3 @@ def build_repository() -> MetadataRepository:
repository = build_repository()

228
memory_gateway/schemas_v2.py Normal file
View File

@ -0,0 +1,228 @@
"""Schemas for the Memory Gateway v2 control-plane API."""
from __future__ import annotations
from datetime import datetime
from enum import Enum
from typing import Any, Literal, Optional
from uuid import uuid4
from pydantic import BaseModel, Field
from .schemas import utc_now
class OperationStatus(str, Enum):
ACCEPTED = "accepted"
RUNNING = "running"
SUCCESS = "success"
PARTIAL_SUCCESS = "partial_success"
FAILED = "failed"
PENDING = "pending"
SKIPPED = "skipped"
class BackendRefStatus(str, Enum):
PENDING = "pending"
SUCCESS = "success"
FAILED = "failed"
SKIPPED = "skipped"
class BackendType(str, Enum):
OPENVIKING = "openviking"
EVERMEMOS = "evermemos"
OBSIDIAN = "obsidian"
class MemoryRefType(str, Enum):
SESSION_ARCHIVE = "session_archive"
CONTEXT_RESOURCE = "context_resource"
MESSAGE_MEMORY = "message_memory"
EPISODIC_MEMORY = "episodic_memory"
PROFILE = "profile"
LONG_TERM_MEMORY = "long_term_memory"
DRAFT_REVIEW = "draft_review"
class TraceContext(BaseModel):
trace_id: Optional[str] = None
span_id: Optional[str] = None
parent_span_id: Optional[str] = None
request_id: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)
class IngestPolicy(BaseModel):
allow_openviking: bool = True
allow_evermemos: bool = True
allow_obsidian_review: bool = False
redact_sensitive: bool = True
require_human_review: bool = False
metadata: dict[str, Any] = Field(default_factory=dict)
class IngestRequest(BaseModel):
workspace_id: str
user_id: str
agent_id: str
session_id: str
turn_id: str
request_id: Optional[str] = None
idempotency_key: Optional[str] = None
namespace: str
source_type: str = "conversation"
source_event_id: Optional[str] = None
role: Literal["system", "user", "assistant", "tool", "agent"] = "user"
content: str
policy: IngestPolicy = Field(default_factory=IngestPolicy)
trace: TraceContext = Field(default_factory=TraceContext)
metadata: dict[str, Any] = Field(default_factory=dict)
class MemoryRef(BaseModel):
id: str = Field(default_factory=lambda: f"ref_{uuid4().hex[:16]}")
gateway_id: str
workspace_id: str
user_id: str
agent_id: Optional[str] = None
session_id: Optional[str] = None
turn_id: Optional[str] = None
namespace: Optional[str] = None
backend_type: BackendType
ref_type: MemoryRefType
native_id: Optional[str] = None
native_uri: Optional[str] = None
provenance_id: Optional[str] = None
idempotency_key: Optional[str] = None
content_hash: Optional[str] = None
source_type: Optional[str] = None
source_event_id: Optional[str] = None
status: BackendRefStatus = BackendRefStatus.PENDING
error_message: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)
created_at: datetime = Field(default_factory=utc_now)
updated_at: datetime = Field(default_factory=utc_now)
class MemoryRefView(MemoryRef):
pass
class IngestResponse(BaseModel):
status: OperationStatus
gateway_id: str
provenance_id: str
request_id: Optional[str] = None
turn_id: str
refs: list[MemoryRefView] = Field(default_factory=list)
errors: list[str] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
class CommitRequest(BaseModel):
workspace_id: str
user_id: str
agent_id: Optional[str] = None
namespace: Optional[str] = None
request_id: Optional[str] = None
idempotency_key: Optional[str] = None
policy: IngestPolicy = Field(default_factory=IngestPolicy)
metadata: dict[str, Any] = Field(default_factory=dict)
class CommitResponse(BaseModel):
status: OperationStatus = OperationStatus.ACCEPTED
job_id: str
session_id: str
message: str = "commit accepted"
refs: list[MemoryRefView] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
class OutboxSummary(BaseModel):
total_events: int = 0
pending_events: int = 0
processing_events: int = 0
success_events: int = 0
skipped_events: int = 0
dead_letter_events: int = 0
class CommitJobView(BaseModel):
job_id: str
workspace_id: str
user_id: str
agent_id: Optional[str] = None
session_id: str
namespace: Optional[str] = None
status: OperationStatus
created_refs_count: int = 0
error_message: Optional[str] = None
created_at: datetime
updated_at: datetime
started_at: Optional[datetime] = None
finished_at: Optional[datetime] = None
outbox_summary: OutboxSummary = Field(default_factory=OutboxSummary)
class OutboxProcessResponse(BaseModel):
status: OperationStatus
worker_id: str
processed_count: int = 0
outbox_summary: OutboxSummary = Field(default_factory=OutboxSummary)
class RetrieveRequest(BaseModel):
workspace_id: str
user_id: str
agent_id: Optional[str] = None
session_id: Optional[str] = None
namespace: Optional[str] = None
query: str
limit: int = Field(default=10, ge=1, le=100)
metadata: dict[str, Any] = Field(default_factory=dict)
class ContextItem(BaseModel):
text: Optional[str] = None
source_backend: BackendType
ref_id: Optional[str] = None
score: float = 0.0
memory_type: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)
class ContextConflict(BaseModel):
ref_ids: list[str] = Field(default_factory=list)
reason: str
metadata: dict[str, Any] = Field(default_factory=dict)
class RetrieveResponse(BaseModel):
status: OperationStatus
items: list[ContextItem] = Field(default_factory=list)
refs: list[MemoryRefView] = Field(default_factory=list)
conflicts: list[ContextConflict] = Field(default_factory=list)
trace_id: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)
class FeedbackRequest(BaseModel):
workspace_id: str
user_id: str
agent_id: Optional[str] = None
session_id: Optional[str] = None
namespace: Optional[str] = None
memory_ref_id: Optional[str] = None
feedback_type: Literal["useful", "not_useful", "incorrect", "duplicate", "outdated", "review_approved", "review_rejected"]
comment: Optional[str] = None
source_type: str = "manual"
source_event_id: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)
class FeedbackResponse(BaseModel):
status: OperationStatus
feedback_id: str
memory_ref_id: Optional[str] = None
metadata: dict[str, Any] = Field(default_factory=dict)

2
memory_gateway/server.py
View File

@ -572,8 +572,10 @@ app.include_router(mcp_router, prefix="/mcp", tags=["mcp"])
# Generic Memory Gateway v1 routes are imported lazily here to avoid changing
# the existing legacy /api and /mcp startup path.
from .api_v1 import router as api_v1_router # noqa: E402
from .api_v2 import router as api_v2_router # noqa: E402
app.include_router(api_v1_router)
app.include_router(api_v2_router)
@app.post("/api/search", dependencies=[Depends(verify_api_key)])

970
memory_gateway/services_v2.py Normal file
View File

@ -0,0 +1,970 @@
"""Service orchestration for the Memory Gateway v2 control plane."""
from __future__ import annotations
import hashlib
from datetime import datetime, timedelta, timezone
from typing import Any, Awaitable, Callable
from uuid import uuid4
from fastapi import HTTPException, status
from .backend_contracts import (
BackendOperation,
BackendCommitResult,
BackendProducedRef,
BackendResultStatus,
BackendWriteResult,
CommitJob,
OutboxEvent,
OutboxEventStatus,
)
from .evermemos_client import EverMemOSClient
from .openviking_client import get_openviking_client
from .repositories import MetadataRepository, repository
from .schemas import AuditLog
from .schemas_v2 import (
BackendRefStatus,
BackendType,
CommitJobView,
CommitRequest,
CommitResponse,
ContextItem,
FeedbackRequest,
FeedbackResponse,
IngestRequest,
IngestResponse,
MemoryRef,
MemoryRefView,
MemoryRefType,
OperationStatus,
OutboxProcessResponse,
OutboxSummary,
RetrieveRequest,
RetrieveResponse,
)
OpenVikingClientFactory = Callable[[], Awaitable[Any]]
class MemoryGatewayV2Service:
def __init__(
self,
repo: MetadataRepository = repository,
openviking_client_factory: OpenVikingClientFactory = get_openviking_client,
evermemos_client: Any | None = None,
) -> None:
self.repo = repo
self.openviking_client_factory = openviking_client_factory
self.evermemos_client = evermemos_client
async def ingest_conversation_turn(self, request: IngestRequest) -> IngestResponse:
normalized = self._normalize_ingest_request(request)
gateway_id = self._build_gateway_id(normalized)
provenance_id = self._build_provenance_id(normalized, gateway_id)
content_hash = self._content_hash(normalized.content)
self._check_namespace_access(normalized)
payload = self._apply_safety_policy(normalized)
refs: list[MemoryRef] = []
if normalized.policy.allow_openviking:
refs.append(
await self._write_openviking_turn(
normalized,
payload,
gateway_id=gateway_id,
provenance_id=provenance_id,
content_hash=content_hash,
)
)
else:
refs.append(
self._save_ref(
normalized,
gateway_id,
provenance_id,
BackendType.OPENVIKING,
MemoryRefType.SESSION_ARCHIVE,
BackendRefStatus.SKIPPED,
content_hash=content_hash,
metadata=self._control_metadata(normalized, content_hash, {"reason": "policy_disabled"}),
)
)
if normalized.policy.allow_evermemos:
refs.append(
await self._write_evermemos_message(
normalized,
payload,
gateway_id=gateway_id,
provenance_id=provenance_id,
content_hash=content_hash,
)
)
else:
refs.append(
self._save_ref(
normalized,
gateway_id,
provenance_id,
BackendType.EVERMEMOS,
MemoryRefType.MESSAGE_MEMORY,
BackendRefStatus.SKIPPED,
content_hash=content_hash,
metadata=self._control_metadata(normalized, content_hash, {"reason": "policy_disabled"}),
)
)
status_value = self._aggregate_ref_status(refs)
errors = [ref.error_message for ref in refs if ref.error_message]
self.repo.add_audit(
AuditLog(
actor_user_id=normalized.user_id,
actor_agent_id=normalized.agent_id,
action="v2_ingest_conversation_turn",
target_type="conversation_turn",
target_id=normalized.turn_id,
namespace=normalized.namespace,
metadata={
"gateway_id": gateway_id,
"provenance_id": provenance_id,
"idempotency_basis": self._idempotency_basis(normalized),
"content_hash": content_hash,
"status": status_value.value,
"source_type": normalized.source_type,
"source_event_id": normalized.source_event_id,
},
)
)
return IngestResponse(
status=status_value,
gateway_id=gateway_id,
provenance_id=provenance_id,
request_id=normalized.request_id,
turn_id=normalized.turn_id,
refs=self._view_refs(refs),
errors=errors,
metadata={"backend_count": len(refs)},
)
async def commit_session(self, session_id: str, request: CommitRequest) -> CommitResponse:
# TODO(v2): add a worker that consumes these outbox events and writes
# resulting backend refs. This method intentionally only records
# control-plane intent.
job_id = f"job_{uuid4().hex[:16]}"
gateway_id = self._commit_gateway_id(session_id, request)
job = CommitJob(
job_id=job_id,
workspace_id=request.workspace_id,
user_id=request.user_id,
agent_id=request.agent_id,
session_id=session_id,
namespace=request.namespace,
requested_by=request.user_id,
)
self.repo.save_commit_job(job)
self._create_commit_outbox_events(gateway_id, job, request)
self.repo.add_audit(
AuditLog(
actor_user_id=request.user_id,
actor_agent_id=request.agent_id,
action="v2_commit_session_accepted",
target_type="session",
target_id=session_id,
namespace=request.namespace,
metadata={
"job_id": job_id,
"gateway_id": gateway_id,
"workspace_id": request.workspace_id,
"outbox_events": 2,
},
)
)
return CommitResponse(
job_id=job_id,
session_id=session_id,
metadata={"gateway_id": gateway_id, "outbox_events": 2},
)
async def retrieve_context(self, request: RetrieveRequest) -> RetrieveResponse:
# TODO(v2): expand namespace ACL, fan out concurrently to OpenViking and
# EverMemOS, then apply lightweight merge/rerank before returning.
refs = self.repo.list_memory_refs(
workspace_id=request.workspace_id,
user_id=request.user_id,
agent_id=request.agent_id,
session_id=request.session_id,
namespace=request.namespace,
limit=request.limit,
)
items = [
ContextItem(
text=None,
source_backend=ref.backend_type,
ref_id=ref.id,
score=0.0,
memory_type=ref.ref_type.value,
metadata={
"status": ref.status.value,
"native_id": ref.native_id,
"native_uri": ref.native_uri,
},
)
for ref in refs
]
trace_id = request.metadata.get("trace_id") if request.metadata else None
return RetrieveResponse(
status=OperationStatus.SUCCESS,
items=items,
refs=self._view_refs(refs),
conflicts=[],
trace_id=trace_id,
metadata={"skeleton": True},
)
async def record_memory_feedback(self, request: FeedbackRequest) -> FeedbackResponse:
# TODO(v2): persist review/feedback state in a dedicated table and route
# accepted corrections back to the owning backend adapter.
feedback_id = f"fb_{uuid4().hex[:16]}"
self.repo.add_audit(
AuditLog(
actor_user_id=request.user_id,
actor_agent_id=request.agent_id,
action=f"v2_feedback:{request.feedback_type}",
target_type="memory_ref",
target_id=request.memory_ref_id,
namespace=request.namespace,
metadata={
"feedback_id": feedback_id,
"workspace_id": request.workspace_id,
"comment": request.comment,
"source_type": request.source_type,
"source_event_id": request.source_event_id,
},
)
)
return FeedbackResponse(
status=OperationStatus.ACCEPTED,
feedback_id=feedback_id,
memory_ref_id=request.memory_ref_id,
)
async def process_pending_outbox_events(
self,
limit: int = 100,
worker_id: str | None = None,
lease_seconds: int = 300,
) -> list[OutboxEvent]:
worker_id = worker_id or f"worker_{uuid4().hex[:12]}"
self.repo.release_expired_processing_events()
events = self.repo.claim_pending_outbox_events(
limit=limit,
worker_id=worker_id,
lease_seconds=lease_seconds,
)
processed: list[OutboxEvent] = []
for event in events:
try:
result = await self._execute_outbox_event(event)
except Exception as exc: # noqa: BLE001
result = BackendCommitResult(
backend_type=event.backend_type,
operation=event.operation,
status=BackendResultStatus.FAILED,
retryable=True,
error_code="adapter_exception",
error_message=str(exc),
)
processed.append(self._apply_outbox_result(event, result))
return processed
async def process_pending_outbox_events_summary(
self,
limit: int = 100,
worker_id: str | None = None,
lease_seconds: int = 300,
) -> OutboxProcessResponse:
worker_id = worker_id or f"worker_{uuid4().hex[:12]}"
processed = await self.process_pending_outbox_events(
limit=limit,
worker_id=worker_id,
lease_seconds=lease_seconds,
)
return OutboxProcessResponse(
status=OperationStatus.SUCCESS,
worker_id=worker_id,
processed_count=len(processed),
outbox_summary=self._outbox_summary(),
)
async def process_commit_job(self, job_id: str) -> CommitJob:
job = self.repo.get_commit_job(job_id)
if not job:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Commit job not found")
self.repo.update_commit_job_status(job_id, OperationStatus.RUNNING.value)
self.repo.release_expired_processing_events()
events = self.repo.list_outbox_events_by_job(job_id)
for event in events:
if event.status == OutboxEventStatus.PENDING:
await self.process_outbox_event(event.id)
events = self.repo.list_outbox_events_by_job(job_id)
final_status = self._aggregate_commit_job_status(events)
created_refs_count = self.repo.count_memory_refs(session_id=job.session_id, status=BackendRefStatus.SUCCESS)
error_message = self._commit_job_error_message(events)
updated = self.repo.update_commit_job_status(
job_id,
final_status.value,
error_message=error_message,
created_refs_count=created_refs_count,
)
if not updated:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Commit job not found")
return updated
def get_commit_job_view(self, job_id: str) -> CommitJobView:
job = self.repo.get_commit_job(job_id)
if not job:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Commit job not found")
return CommitJobView(
job_id=job.job_id,
workspace_id=job.workspace_id,
user_id=job.user_id,
agent_id=job.agent_id,
session_id=job.session_id,
namespace=job.namespace,
status=job.status,
created_refs_count=job.created_refs_count,
error_message=job.error_message,
created_at=job.created_at,
updated_at=job.updated_at,
started_at=job.started_at,
finished_at=job.finished_at,
outbox_summary=self._outbox_summary(self.repo.list_outbox_events_by_job(job_id)),
)
async def process_outbox_event(self, event_id: str) -> OutboxEvent | None:
event = self.repo.claim_outbox_event(event_id)
if not event:
return None
try:
result = await self._execute_outbox_event(event)
except Exception as exc: # noqa: BLE001
result = BackendCommitResult(
backend_type=event.backend_type,
operation=event.operation,
status=BackendResultStatus.FAILED,
retryable=True,
error_code="adapter_exception",
error_message=str(exc),
)
return self._apply_outbox_result(event, result)
def list_memory_refs(
self,
workspace_id: str | None = None,
user_id: str | None = None,
agent_id: str | None = None,
session_id: str | None = None,
namespace: str | None = None,
backend_type: BackendType | str | None = None,
ref_type: MemoryRefType | str | None = None,
status: BackendRefStatus | str | None = None,
limit: int = 100,
) -> list[MemoryRef]:
return self.repo.list_memory_refs(
workspace_id=workspace_id,
user_id=user_id,
agent_id=agent_id,
session_id=session_id,
namespace=namespace,
backend_type=backend_type,
ref_type=ref_type,
status=status,
limit=limit,
)
async def _execute_outbox_event(self, event: OutboxEvent) -> BackendCommitResult | BackendWriteResult:
payload = self._outbox_payload(event)
if event.operation != BackendOperation.COMMIT_SESSION:
return BackendCommitResult(
backend_type=event.backend_type,
operation=event.operation,
status=BackendResultStatus.SKIPPED,
metadata={"reason": "unsupported_operation"},
)
if event.backend_type == BackendType.OPENVIKING:
client = await self.openviking_client_factory()
if not hasattr(client, "commit_session_v2"):
return BackendCommitResult(
backend_type=BackendType.OPENVIKING,
operation=BackendOperation.COMMIT_SESSION,
status=BackendResultStatus.SKIPPED,
metadata={"reason": "adapter_method_missing"},
)
result = await client.commit_session_v2(payload)
return result
if event.backend_type == BackendType.EVERMEMOS:
client = self.evermemos_client or EverMemOSClient()
if not hasattr(client, "extract_profile_long_term_v2"):
return BackendCommitResult(
backend_type=BackendType.EVERMEMOS,
operation=BackendOperation.COMMIT_SESSION,
status=BackendResultStatus.SKIPPED,
metadata={"reason": "adapter_method_missing"},
)
result = client.extract_profile_long_term_v2(payload)
if hasattr(result, "__await__"):
result = await result
return result
return BackendCommitResult(
backend_type=event.backend_type,
operation=event.operation,
status=BackendResultStatus.SKIPPED,
metadata={"reason": "unsupported_backend"},
)
def _outbox_payload(self, event: OutboxEvent) -> dict[str, Any]:
return {
"event_id": event.id,
"gateway_id": event.gateway_id,
"workspace_id": event.workspace_id,
"user_id": event.user_id,
"agent_id": event.agent_id,
"session_id": event.session_id,
"backend_type": event.backend_type.value,
"operation": event.operation.value,
"payload_ref": event.payload_ref,
"metadata": self._safe_control_metadata(event.metadata),
}
def _apply_outbox_result(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> OutboxEvent:
result_data = self._backend_result_to_dict(result)
safe_result = self._backend_control_metadata(result_data)
event.metadata = self._safe_control_metadata({**event.metadata, "backend_result": safe_result})
event.last_error = result.error_message
event.updated_at = datetime.now(timezone.utc)
if result.status == BackendResultStatus.SUCCESS:
self._write_commit_memory_refs(event, result)
event.status = OutboxEventStatus.SUCCESS
self._clear_outbox_lock(event)
return self.repo.save_outbox_event(event)
if result.status == BackendResultStatus.SKIPPED:
event.status = OutboxEventStatus.SKIPPED
self._clear_outbox_lock(event)
return self.repo.save_outbox_event(event)
if result.status == BackendResultStatus.FAILED:
return self._handle_failed_outbox_event(event, result)
event.status = OutboxEventStatus.PENDING
self._clear_outbox_lock(event)
return self.repo.save_outbox_event(event)
def _handle_failed_outbox_event(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> OutboxEvent:
event.attempt_count += 1
event.last_error = result.error_message
if result.retryable and event.attempt_count < event.max_attempts:
event.status = OutboxEventStatus.PENDING
event.next_retry_at = datetime.now(timezone.utc) + timedelta(seconds=min(60, 2**event.attempt_count))
else:
event.status = OutboxEventStatus.DEAD_LETTER
event.next_retry_at = None
self._clear_outbox_lock(event)
return self.repo.save_outbox_event(event)
def _clear_outbox_lock(self, event: OutboxEvent) -> None:
event.locked_by = None
event.locked_at = None
event.lease_expires_at = None
def _write_commit_memory_refs(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> list[MemoryRef]:
produced_refs = result.refs if isinstance(result, BackendCommitResult) and result.refs else []
if produced_refs:
refs: list[MemoryRef] = []
for index, produced_ref in enumerate(produced_refs):
saved = self._write_commit_memory_ref(event, result, produced_ref, index=index)
if saved:
refs.append(saved)
return refs
fallback = BackendProducedRef(
ref_type=self._commit_ref_type(event, result),
native_id=result.native_id,
native_uri=result.native_uri,
metadata={},
)
saved = self._write_commit_memory_ref(event, result, fallback, index=0)
return [saved] if saved else []
def _write_commit_memory_ref(
self,
event: OutboxEvent,
result: BackendCommitResult | BackendWriteResult,
produced_ref: BackendProducedRef,
index: int,
) -> MemoryRef | None:
stable_key = self._produced_ref_stable_key(produced_ref, index)
if not produced_ref.native_id and not produced_ref.native_uri and not stable_key:
return None
ref_type = produced_ref.ref_type
ref_id = self._memory_ref_id(event.gateway_id, event.backend_type, ref_type, produced_ref.native_id, produced_ref.native_uri, stable_key)
existing = self.repo.get_memory_ref(ref_id)
safe_produced_metadata = self._safe_control_metadata(produced_ref.metadata)
ref = MemoryRef(
id=ref_id,
gateway_id=event.gateway_id,
workspace_id=event.workspace_id,
user_id=event.user_id,
agent_id=event.agent_id,
session_id=event.session_id,
namespace=event.metadata.get("namespace"),
backend_type=event.backend_type,
ref_type=ref_type,
native_id=produced_ref.native_id,
native_uri=produced_ref.native_uri,
provenance_id="prov_"
+ hashlib.sha256(f"{event.id}|{ref_type.value}|{produced_ref.native_id}|{produced_ref.native_uri}".encode("utf-8")).hexdigest()[:24],
source_type="commit_session",
source_event_id=event.id,
status=BackendRefStatus.SUCCESS,
error_message=None,
metadata={
"schema_version": "memory-gateway.commit-ref.v2",
"job_id": self._job_id_from_payload_ref(event.payload_ref),
"outbox_event_id": event.id,
"operation": event.operation.value,
"produced_ref_index": index,
"stable_key": stable_key,
"produced_ref": safe_produced_metadata,
"backend_result": self._backend_control_metadata(self._backend_result_to_dict(result)),
},
)
if existing:
ref.created_at = existing.created_at
return self.repo.save_memory_ref(ref)
def _commit_ref_type(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> MemoryRefType:
requested = result.metadata.get("ref_type") if isinstance(result.metadata, dict) else None
if requested:
try:
return MemoryRefType(requested)
except ValueError:
pass
if event.backend_type == BackendType.OPENVIKING:
return MemoryRefType.SESSION_ARCHIVE
if event.backend_type == BackendType.EVERMEMOS:
return MemoryRefType.LONG_TERM_MEMORY
return MemoryRefType.DRAFT_REVIEW
def _job_id_from_payload_ref(self, payload_ref: str | None) -> str | None:
if payload_ref and payload_ref.startswith("commit_job:"):
return payload_ref.split(":", 1)[1]
return None
def _aggregate_commit_job_status(self, events: list[OutboxEvent]) -> OperationStatus:
if not events:
return OperationStatus.FAILED
statuses = {event.status for event in events}
if statuses.issubset({OutboxEventStatus.SUCCESS, OutboxEventStatus.SKIPPED}):
return OperationStatus.SUCCESS
if statuses == {OutboxEventStatus.DEAD_LETTER} or statuses == {OutboxEventStatus.FAILED}:
return OperationStatus.FAILED
if OutboxEventStatus.PENDING in statuses or OutboxEventStatus.PROCESSING in statuses:
if OutboxEventStatus.SUCCESS in statuses:
return OperationStatus.PARTIAL_SUCCESS
return OperationStatus.FAILED
if OutboxEventStatus.SUCCESS in statuses:
return OperationStatus.PARTIAL_SUCCESS
return OperationStatus.FAILED
def _commit_job_error_message(self, events: list[OutboxEvent]) -> str | None:
errors = [event.last_error for event in events if event.last_error]
return "; ".join(errors) if errors else None
def _outbox_summary(self, events: list[OutboxEvent] | None = None) -> OutboxSummary:
events = events if events is not None else self.repo.list_outbox_events(limit=100000)
counts = {status: 0 for status in OutboxEventStatus}
for event in events:
counts[event.status] = counts.get(event.status, 0) + 1
return OutboxSummary(
total_events=len(events),
pending_events=counts.get(OutboxEventStatus.PENDING, 0),
processing_events=counts.get(OutboxEventStatus.PROCESSING, 0),
success_events=counts.get(OutboxEventStatus.SUCCESS, 0),
skipped_events=counts.get(OutboxEventStatus.SKIPPED, 0),
dead_letter_events=counts.get(OutboxEventStatus.DEAD_LETTER, 0),
)
def _safe_control_metadata(self, metadata: dict[str, Any] | None) -> dict[str, Any]:
if not metadata:
return {}
blocked = {"content", "raw_request", "messages", "conversation", "transcript"}
safe: dict[str, Any] = {}
for key, value in metadata.items():
if key in blocked:
continue
if isinstance(value, dict):
nested = self._safe_control_metadata(value)
if nested:
safe[key] = nested
elif isinstance(value, (str, int, float, bool)) or value is None:
safe[key] = value
elif isinstance(value, list):
safe[key] = [item for item in value if isinstance(item, (str, int, float, bool))]
return safe
def _normalize_ingest_request(self, request: IngestRequest) -> IngestRequest:
data = request.model_copy(deep=True)
data.namespace = data.namespace.strip("/")
data.content = data.content.strip()
if not data.namespace:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="namespace is required")
if not data.content:
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="content is required")
return data
def _build_gateway_id(self, request: IngestRequest) -> str:
seed = self._idempotency_basis(request)
return "gw_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]
def _build_provenance_id(self, request: IngestRequest, gateway_id: str) -> str:
seed = f"{gateway_id}|{self._idempotency_basis(request)}"
return "prov_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]
def _idempotency_basis(self, request: IngestRequest) -> str:
if request.idempotency_key:
return f"idempotency_key:{request.workspace_id}:{request.idempotency_key}"
if request.source_event_id:
return f"source_event_id:{request.workspace_id}:{request.source_type}:{request.source_event_id}"
return f"turn:{request.workspace_id}:{request.session_id}:{request.turn_id}"
def _content_hash(self, content: str) -> str:
return hashlib.sha256(content.encode("utf-8")).hexdigest()
def _check_namespace_access(self, request: IngestRequest) -> None:
# TODO(v2): enforce workspace/user/agent namespace ACL tree.
if not request.workspace_id or not request.user_id or not request.agent_id:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="workspace, user, and agent are required")
def _apply_safety_policy(self, request: IngestRequest) -> dict[str, Any]:
# TODO(v2): apply configurable redaction/safety rules before fan-out.
return {
"workspace_id": request.workspace_id,
"user_id": request.user_id,
"agent_id": request.agent_id,
"session_id": request.session_id,
"turn_id": request.turn_id,
"namespace": request.namespace,
"source_type": request.source_type,
"source_event_id": request.source_event_id,
"role": request.role,
"content": request.content,
"metadata": request.metadata,
"trace": request.trace.model_dump(mode="json"),
}
async def _write_openviking_turn(
self,
request: IngestRequest,
payload: dict[str, Any],
gateway_id: str,
provenance_id: str,
content_hash: str,
) -> MemoryRef:
try:
client = await self.openviking_client_factory()
if not hasattr(client, "ingest_conversation_turn"):
return self._save_ref(
request,
gateway_id,
provenance_id,
BackendType.OPENVIKING,
MemoryRefType.SESSION_ARCHIVE,
BackendRefStatus.SKIPPED,
content_hash=content_hash,
metadata=self._control_metadata(request, content_hash, {"reason": "adapter_method_missing"}),
)
result = await client.ingest_conversation_turn(payload)
return self._ref_from_backend_result(
request,
gateway_id,
provenance_id,
BackendType.OPENVIKING,
MemoryRefType.SESSION_ARCHIVE,
result,
content_hash,
)
except Exception as exc: # noqa: BLE001
return self._save_ref(
request,
gateway_id,
provenance_id,
BackendType.OPENVIKING,
MemoryRefType.SESSION_ARCHIVE,
BackendRefStatus.FAILED,
content_hash=content_hash,
error_message=str(exc),
metadata=self._control_metadata(request, content_hash),
)
async def _write_evermemos_message(
self,
request: IngestRequest,
payload: dict[str, Any],
gateway_id: str,
provenance_id: str,
content_hash: str,
) -> MemoryRef:
try:
client = self.evermemos_client or EverMemOSClient()
if not hasattr(client, "ingest_message"):
return self._save_ref(
request,
gateway_id,
provenance_id,
BackendType.EVERMEMOS,
MemoryRefType.MESSAGE_MEMORY,
BackendRefStatus.SKIPPED,
content_hash=content_hash,
metadata=self._control_metadata(request, content_hash, {"reason": "adapter_method_missing"}),
)
result = client.ingest_message(payload)
if hasattr(result, "__await__"):
result = await result
return self._ref_from_backend_result(
request,
gateway_id,
provenance_id,
BackendType.EVERMEMOS,
MemoryRefType.MESSAGE_MEMORY,
result,
content_hash,
)
except Exception as exc: # noqa: BLE001
return self._save_ref(
request,
gateway_id,
provenance_id,
BackendType.EVERMEMOS,
MemoryRefType.MESSAGE_MEMORY,
BackendRefStatus.FAILED,
content_hash=content_hash,
error_message=str(exc),
metadata=self._control_metadata(request, content_hash),
)
def _ref_from_backend_result(
self,
request: IngestRequest,
gateway_id: str,
provenance_id: str,
backend_type: BackendType,
ref_type: MemoryRefType,
result: Any,
content_hash: str,
) -> MemoryRef:
data = self._backend_result_to_dict(result)
raw_status = str(data.get("status") or "success")
ref_status = BackendRefStatus.SUCCESS
if raw_status in {BackendRefStatus.PENDING.value, BackendRefStatus.FAILED.value, BackendRefStatus.SKIPPED.value}:
ref_status = BackendRefStatus(raw_status)
native_id = data.get("native_id") or data.get("id") or data.get("memory_id") or data.get("session_id")
native_uri = data.get("native_uri") or data.get("uri") or data.get("url")
return self._save_ref(
request,
gateway_id,
provenance_id,
backend_type,
ref_type,
ref_status,
native_id=native_id,
native_uri=native_uri,
error_message=data.get("error") or data.get("error_message"),
content_hash=content_hash,
metadata=self._control_metadata(
request,
content_hash,
{"backend_response": self._backend_control_metadata(data)},
),
)
def _save_ref(
self,
request: IngestRequest,
gateway_id: str,
provenance_id: str,
backend_type: BackendType,
ref_type: MemoryRefType,
ref_status: BackendRefStatus,
native_id: str | None = None,
native_uri: str | None = None,
error_message: str | None = None,
content_hash: str | None = None,
metadata: dict[str, Any] | None = None,
) -> MemoryRef:
ref_id = self._memory_ref_id(gateway_id, backend_type, ref_type)
existing = self.repo.get_memory_ref(ref_id)
ref = MemoryRef(
id=ref_id,
gateway_id=gateway_id,
workspace_id=request.workspace_id,
user_id=request.user_id,
agent_id=request.agent_id,
session_id=request.session_id,
turn_id=request.turn_id,
namespace=request.namespace,
backend_type=backend_type,
ref_type=ref_type,
native_id=native_id,
native_uri=native_uri,
provenance_id=provenance_id,
idempotency_key=request.idempotency_key,
content_hash=content_hash,
source_type=request.source_type,
source_event_id=request.source_event_id,
status=ref_status,
error_message=error_message,
metadata=metadata or {},
)
if existing:
ref.created_at = existing.created_at
return self.repo.save_memory_ref(ref)
def _memory_ref_id(
self,
gateway_id: str,
backend_type: BackendType,
ref_type: MemoryRefType,
native_id: str | None = None,
native_uri: str | None = None,
stable_key: str | None = None,
) -> str:
native_key = f"|{native_id or ''}|{native_uri or ''}|{stable_key or ''}" if native_id or native_uri or stable_key else ""
seed = f"{gateway_id}|{backend_type.value}|{ref_type.value}{native_key}"
return "ref_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]
def _produced_ref_stable_key(self, produced_ref: BackendProducedRef, index: int) -> str | None:
for key in ("stable_key", "backend_ref_key", "idempotency_key"):
value = produced_ref.metadata.get(key)
if isinstance(value, str) and value:
return value
if not produced_ref.native_id and not produced_ref.native_uri:
return f"produced_ref_index:{index}"
return None
def _aggregate_ref_status(self, refs: list[MemoryRef]) -> OperationStatus:
if not refs:
return OperationStatus.SKIPPED
statuses = {ref.status for ref in refs}
if statuses == {BackendRefStatus.SUCCESS}:
return OperationStatus.SUCCESS
if BackendRefStatus.SUCCESS in statuses and (BackendRefStatus.FAILED in statuses or BackendRefStatus.SKIPPED in statuses):
return OperationStatus.PARTIAL_SUCCESS
if statuses == {BackendRefStatus.FAILED}:
return OperationStatus.FAILED
if BackendRefStatus.FAILED in statuses:
return OperationStatus.PARTIAL_SUCCESS
if statuses == {BackendRefStatus.SKIPPED}:
return OperationStatus.SKIPPED
return OperationStatus.PENDING
def _view_refs(self, refs: list[MemoryRef]) -> list[MemoryRefView]:
return [MemoryRefView.model_validate(ref.model_dump(mode="json")) for ref in refs]
def _backend_control_metadata(self, data: dict[str, Any]) -> dict[str, Any]:
allowed_keys = {
"status",
"reason",
"native_id",
"native_uri",
"id",
"uri",
"url",
"memory_id",
"session_id",
"retryable",
"error_code",
"error",
"error_message",
"latency_ms",
}
metadata = {key: value for key, value in data.items() if key in allowed_keys}
nested_metadata = data.get("metadata")
if isinstance(nested_metadata, dict):
for key in ("reason", "backend_request_id", "latency_ms", "schema_version"):
if key in nested_metadata:
metadata[key] = nested_metadata[key]
return metadata
def _control_metadata(
self,
request: IngestRequest,
content_hash: str,
extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
metadata: dict[str, Any] = {
"schema_version": "memory-gateway.control-ref.v2",
"idempotency_basis": self._idempotency_basis(request),
"content_hash": content_hash,
}
source_channel = request.metadata.get("source_channel") or request.metadata.get("channel")
if source_channel:
metadata["source_channel"] = source_channel
if request.trace.trace_id:
metadata["trace_id"] = request.trace.trace_id
if request.trace.request_id:
metadata["trace_request_id"] = request.trace.request_id
if extra:
metadata.update(extra)
return metadata
def _backend_result_to_dict(self, result: Any) -> dict[str, Any]:
if isinstance(result, BackendWriteResult):
data = result.model_dump(mode="json")
if result.status == BackendResultStatus.FAILED and result.retryable:
data["retryable"] = True
return data
if hasattr(result, "model_dump"):
return result.model_dump(mode="json")
return result if isinstance(result, dict) else {"raw": str(result)}
def _commit_gateway_id(self, session_id: str, request: CommitRequest) -> str:
basis = request.idempotency_key or request.request_id or session_id
seed = f"commit:{request.workspace_id}:{session_id}:{basis}"
return "gwc_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]
def _create_commit_outbox_events(self, gateway_id: str, job: CommitJob, request: CommitRequest) -> None:
metadata = {
"schema_version": "memory-gateway.outbox.v2",
"job_id": job.job_id,
"namespace": request.namespace,
"idempotency_key": request.idempotency_key,
"request_id": request.request_id,
}
for backend_type in (BackendType.OPENVIKING, BackendType.EVERMEMOS):
event = OutboxEvent(
id=self._outbox_event_id(gateway_id, backend_type, BackendOperation.COMMIT_SESSION),
event_type="commit_session",
gateway_id=gateway_id,
workspace_id=request.workspace_id,
user_id=request.user_id,
agent_id=request.agent_id,
session_id=job.session_id,
backend_type=backend_type,
operation=BackendOperation.COMMIT_SESSION,
payload_ref=f"commit_job:{job.job_id}",
metadata=metadata,
)
self.repo.save_outbox_event(event)
def _outbox_event_id(self, gateway_id: str, backend_type: BackendType, operation: BackendOperation) -> str:
seed = f"{gateway_id}|{backend_type.value}|{operation.value}"
return "outbox_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]
v2_service = MemoryGatewayV2Service()

9
memory_gateway/types.py
View File

@ -12,18 +12,25 @@ class ServerConfig(BaseModel):
class OpenVikingConfig(BaseModel):
"""OpenViking 后端配置"""
enabled: bool = False
mode: Literal["offline", "skeleton", "real"] = "offline"
url: str = "http://localhost:1933"
api_key: str = ""
timeout: int = 30
verify_ssl: bool = True
ingest_path: str = "/api/v1/sessions/{session_id}/messages"
class EverMemOSConfig(BaseModel):
"""External EverMemOS consolidation service configuration."""
enabled: bool = True
enabled: bool = False
mode: Literal["offline", "skeleton", "real"] = "offline"
url: str = "http://127.0.0.1:1995"
api_key: str = ""
timeout: int = 30
verify_ssl: bool = True
health_path: str = "/health"
ingest_path: str = "/api/v1/memories"
consolidate_path: str = "/v1/sessions/consolidate"
fallback_to_local: bool = True

49
memory_gateway/worker_v2.py Normal file
View File

@ -0,0 +1,49 @@
"""Lightweight v2 outbox worker entrypoint.
Usage:
python -m memory_gateway.worker_v2 --limit 100 --worker-id local-worker --lease-seconds 300
"""
from __future__ import annotations
import argparse
import asyncio
import json
from typing import Sequence
from uuid import uuid4
from .services_v2 import v2_service
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Process Memory Gateway v2 outbox events once.")
parser.add_argument("--limit", type=int, default=100, help="Maximum pending events to claim and process.")
parser.add_argument("--worker-id", default=None, help="Stable worker id recorded in outbox lease fields.")
parser.add_argument("--lease-seconds", type=int, default=300, help="Lease duration for claimed events.")
return parser
async def run_once(limit: int, worker_id: str | None, lease_seconds: int) -> dict[str, object]:
worker_id = worker_id or f"worker_{uuid4().hex[:12]}"
response = await v2_service.process_pending_outbox_events_summary(
limit=limit,
worker_id=worker_id,
lease_seconds=lease_seconds,
)
return response.model_dump(mode="json")
def main(argv: Sequence[str] | None = None) -> int:
args = build_parser().parse_args(argv)
payload = asyncio.run(
run_once(
limit=args.limit,
worker_id=args.worker_id,
lease_seconds=args.lease_seconds,
)
)
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())