diff --git a/docs/openviking_adapter_config.md b/docs/openviking_adapter_config.md new file mode 100644 index 0000000..0fa68e6 --- /dev/null +++ b/docs/openviking_adapter_config.md @@ -0,0 +1,93 @@ +# OpenViking Adapter Config + +## Overview + +Memory Gateway v2 keeps the OpenViking ingest adapter in `offline` / `skeleton` +mode by default. In the default configuration it does not send any HTTP +requests. + +## Modes + +### Offline + +`mode: offline` + +The adapter must not touch the network. It returns fixture-backed normalized +results through the existing skeleton path. + +### Skeleton + +`mode: skeleton` + +This behaves like `offline` for now. It keeps the same normalized result path +without sending HTTP requests. + +### Real + +Real mode is enabled only when: + +- `mode: real` + +When real mode is active, the adapter may send an HTTP request for OpenViking +ingest only. Commit and retrieve remain offline/skeleton in the current phase. +The legacy `enabled` field is retained for config compatibility, but it does +not open the network path by itself. + +## Config Fields + +- `base_url` + The OpenViking API base URL. +- `api_key` + Token used only for request headers. +- `timeout` + Request timeout in seconds. +- `verify_ssl` + TLS verification toggle for the real HTTP path. +- `ingest_path` + Configurable ingest endpoint path template. The current placeholder is + `/api/v1/sessions/{session_id}/messages`. + +## Example Config + +### Offline Example + +```yaml +openviking: + enabled: false + mode: offline + url: http://localhost:1933 + timeout: 30 + verify_ssl: true +``` + +### Real Example + +```yaml +openviking: + enabled: false + mode: real + url: https://openviking.example.internal + api_key: YOUR_OPENVIKING_TOKEN + timeout: 30 + verify_ssl: true + ingest_path: /api/v1/sessions/{session_id}/messages +``` + +## Security + +Runtime ingest requests may temporarily include `content` while the current +request is in flight. Memory Gateway does not persist `content`, +`raw_request`, `messages`, or `transcript` into SQLite metadata, outbox +payloads, or audit summaries. + +`api_key` / tokens are used only in request headers. They do not belong in: + +- adapter result metadata +- audit summaries +- persisted MemoryRef metadata +- error messages + +## Notes + +The current ingest endpoint path is still a configurable placeholder. It should +be calibrated once the real OpenViking API contract is stable. diff --git a/memory_gateway/api_v2.py b/memory_gateway/api_v2.py new file mode 100644 index 0000000..dff080c --- /dev/null +++ b/memory_gateway/api_v2.py @@ -0,0 +1,90 @@ +"""Memory Gateway v2 workflow API.""" +from __future__ import annotations + +from typing import Optional + +from fastapi import APIRouter, Depends, Query + +from .schemas_v2 import ( + BackendRefStatus, + BackendType, + CommitJobView, + CommitRequest, + CommitResponse, + FeedbackRequest, + FeedbackResponse, + IngestRequest, + IngestResponse, + MemoryRefType, + MemoryRefView, + OutboxProcessResponse, + RetrieveRequest, + RetrieveResponse, +) +from .server_auth import verify_api_key_compat +from .services_v2 import v2_service + +router = APIRouter(prefix="/v2", tags=["memory-v2"], dependencies=[Depends(verify_api_key_compat)]) + + +@router.post("/conversations/ingest", response_model=IngestResponse) +async def ingest_conversation(request: IngestRequest): + return await v2_service.ingest_conversation_turn(request) + + +@router.post("/conversations/{session_id}/commit", response_model=CommitResponse) +async def commit_conversation(session_id: str, request: CommitRequest): + return await v2_service.commit_session(session_id, request) + + +@router.get("/jobs/{job_id}", response_model=CommitJobView) +async def get_commit_job(job_id: str): + return v2_service.get_commit_job_view(job_id) + + +@router.post("/context/retrieve", response_model=RetrieveResponse) +async def retrieve_context(request: RetrieveRequest): + return await v2_service.retrieve_context(request) + + +@router.get("/memory/refs", response_model=list[MemoryRefView]) +async def list_memory_refs( + workspace_id: Optional[str] = Query(default=None), + user_id: Optional[str] = Query(default=None), + agent_id: Optional[str] = Query(default=None), + session_id: Optional[str] = Query(default=None), + namespace: Optional[str] = Query(default=None), + backend_type: Optional[BackendType] = Query(default=None), + ref_type: Optional[MemoryRefType] = Query(default=None), + status: Optional[BackendRefStatus] = Query(default=None), + limit: int = Query(default=100, ge=1, le=1000), +): + return v2_service.list_memory_refs( + workspace_id=workspace_id, + user_id=user_id, + agent_id=agent_id, + session_id=session_id, + namespace=namespace, + backend_type=backend_type, + ref_type=ref_type, + status=status, + limit=limit, + ) + + +@router.post("/memory/feedback", response_model=FeedbackResponse) +async def memory_feedback(request: FeedbackRequest): + return await v2_service.record_memory_feedback(request) + + +@router.post("/admin/outbox/process", response_model=OutboxProcessResponse, tags=["memory-v2-admin"]) +async def process_outbox( + limit: int = Query(default=100, ge=1, le=1000), + worker_id: Optional[str] = Query(default=None), + lease_seconds: int = Query(default=300, ge=1, le=3600), +): + return await v2_service.process_pending_outbox_events_summary( + limit=limit, + worker_id=worker_id, + lease_seconds=lease_seconds, + ) diff --git a/memory_gateway/backend_adapter_mapping.py b/memory_gateway/backend_adapter_mapping.py new file mode 100644 index 0000000..e59feeb --- /dev/null +++ b/memory_gateway/backend_adapter_mapping.py @@ -0,0 +1,129 @@ +"""Contract-first mapping spec for future v2 backend adapters. + +This module intentionally does not call OpenViking, EverMemOS, or Obsidian. +It documents the stable Gateway control-plane fields that may be persisted in +outbox payload refs, SQLite metadata_json, audit summaries, and related control +records. It is not a validator for transient runtime adapter request objects: +the Gateway may pass conversation content to a backend during the current +request lifecycle, but must not persist that content in SQLite/outbox/audit +control-plane records. +""" +from __future__ import annotations + +from typing import Final, NamedTuple + +from .backend_contracts import BackendCommitResult, BackendOperation, BackendRetrieveResult, BackendWriteResult +from .schemas_v2 import BackendType + + +CONTROL_PLANE_PERSISTED_PAYLOAD_FIELDS: Final[frozenset[str]] = frozenset( + { + "event_id", + "gateway_id", + "workspace_id", + "user_id", + "agent_id", + "session_id", + "turn_id", + "namespace", + "source_type", + "source_event_id", + "backend_type", + "operation", + "payload_ref", + "metadata", + "trace", + "role", + } +) +CONTROL_PLANE_PAYLOAD_FIELDS: Final[frozenset[str]] = CONTROL_PLANE_PERSISTED_PAYLOAD_FIELDS + +DISALLOWED_PAYLOAD_FIELDS: Final[frozenset[str]] = frozenset( + { + "content", + "raw_request", + "messages", + "conversation", + "transcript", + } +) + + +class AdapterMappingSpec(NamedTuple): + backend_type: BackendType + operation: BackendOperation + adapter_method: str + backend_capability: str + result_model: type[BackendWriteResult] | type[BackendCommitResult] | type[BackendRetrieveResult] + allowed_payload_fields: frozenset[str] = CONTROL_PLANE_PERSISTED_PAYLOAD_FIELDS + + +ADAPTER_MAPPING_SPECS: Final[tuple[AdapterMappingSpec, ...]] = ( + AdapterMappingSpec( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.INGEST_TURN, + adapter_method="ingest_conversation_turn", + backend_capability="session archive append / resource context organization", + result_model=BackendWriteResult, + ), + AdapterMappingSpec( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.COMMIT_SESSION, + adapter_method="commit_session_v2", + backend_capability="session commit and session archive ref creation", + result_model=BackendCommitResult, + ), + AdapterMappingSpec( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.RETRIEVE_CONTEXT, + adapter_method="retrieve_context_v2", + backend_capability="runtime session/resource context retrieval", + result_model=BackendRetrieveResult, + ), + AdapterMappingSpec( + backend_type=BackendType.EVERMEMOS, + operation=BackendOperation.INGEST_TURN, + adapter_method="ingest_message", + backend_capability="message-level memory ingestion", + result_model=BackendWriteResult, + ), + AdapterMappingSpec( + backend_type=BackendType.EVERMEMOS, + operation=BackendOperation.COMMIT_SESSION, + adapter_method="extract_profile_long_term_v2", + backend_capability="episodic/profile/long-term extraction", + result_model=BackendCommitResult, + ), + AdapterMappingSpec( + backend_type=BackendType.EVERMEMOS, + operation=BackendOperation.RETRIEVE_CONTEXT, + adapter_method="retrieve_context_v2", + backend_capability="episodic/profile/long-term memory retrieval", + result_model=BackendRetrieveResult, + ), + AdapterMappingSpec( + backend_type=BackendType.OBSIDIAN, + operation=BackendOperation.CREATE_REVIEW_DRAFT, + adapter_method="create_review_draft_v2", + backend_capability="human review draft creation for high-risk/high-conflict candidates", + result_model=BackendWriteResult, + ), +) + + +def get_adapter_mapping_spec(backend_type: BackendType, operation: BackendOperation) -> AdapterMappingSpec: + for spec in ADAPTER_MAPPING_SPECS: + if spec.backend_type == backend_type and spec.operation == operation: + return spec + raise KeyError(f"No v2 adapter mapping for {backend_type.value}:{operation.value}") + + +def validate_control_plane_payload(payload: dict[str, object]) -> None: + """Validate only persisted control-plane payloads, not runtime adapter requests.""" + blocked = sorted(DISALLOWED_PAYLOAD_FIELDS.intersection(payload)) + if blocked: + raise ValueError(f"Control-plane persisted payload includes disallowed content fields: {', '.join(blocked)}") + + +def validate_control_plane_persisted_payload(payload: dict[str, object]) -> None: + validate_control_plane_payload(payload) diff --git a/memory_gateway/backend_contracts.py b/memory_gateway/backend_contracts.py new file mode 100644 index 0000000..4d8554f --- /dev/null +++ b/memory_gateway/backend_contracts.py @@ -0,0 +1,134 @@ +"""Backend adapter contracts for Memory Gateway v2.""" +from __future__ import annotations + +from datetime import datetime +from enum import Enum +from typing import Any, Optional +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from .schemas import utc_now +from .schemas_v2 import BackendType, MemoryRefType, OperationStatus + + +class BackendOperation(str, Enum): + INGEST_TURN = "ingest_turn" + COMMIT_SESSION = "commit_session" + RETRIEVE_CONTEXT = "retrieve_context" + CREATE_REVIEW_DRAFT = "create_review_draft" + + +class BackendResultStatus(str, Enum): + SUCCESS = "success" + FAILED = "failed" + SKIPPED = "skipped" + PENDING = "pending" + + +class OutboxEventStatus(str, Enum): + PENDING = "pending" + PROCESSING = "processing" + SUCCESS = "success" + SKIPPED = "skipped" + FAILED = "failed" + DEAD_LETTER = "dead_letter" + + +class BackendWriteResult(BaseModel): + backend_type: BackendType + operation: BackendOperation + status: BackendResultStatus + native_id: Optional[str] = None + native_uri: Optional[str] = None + retryable: bool = False + error_code: Optional[str] = None + error_message: Optional[str] = None + latency_ms: Optional[float] = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class BackendProducedRef(BaseModel): + ref_type: MemoryRefType + native_id: Optional[str] = None + native_uri: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class BackendCommitResult(BaseModel): + backend_type: BackendType + operation: BackendOperation = BackendOperation.COMMIT_SESSION + status: BackendResultStatus + native_id: Optional[str] = None + native_uri: Optional[str] = None + retryable: bool = False + error_code: Optional[str] = None + error_message: Optional[str] = None + latency_ms: Optional[float] = None + created_refs: list[str] = Field(default_factory=list) + refs: list[BackendProducedRef] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class BackendRetrieveItem(BaseModel): + text: Optional[str] = None + source_backend: BackendType + ref_id: Optional[str] = None + score: float = 0.0 + memory_type: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class BackendRetrieveResult(BaseModel): + backend_type: BackendType + operation: BackendOperation = BackendOperation.RETRIEVE_CONTEXT + status: BackendResultStatus + native_id: Optional[str] = None + native_uri: Optional[str] = None + retryable: bool = False + error_code: Optional[str] = None + error_message: Optional[str] = None + latency_ms: Optional[float] = None + items: list[BackendRetrieveItem] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class OutboxEvent(BaseModel): + id: str = Field(default_factory=lambda: f"outbox_{uuid4().hex[:16]}") + event_type: str + gateway_id: str + workspace_id: str + user_id: str + agent_id: Optional[str] = None + session_id: Optional[str] = None + backend_type: BackendType + operation: BackendOperation + payload_ref: Optional[str] = None + status: OutboxEventStatus = OutboxEventStatus.PENDING + attempt_count: int = 0 + max_attempts: int = 3 + next_retry_at: Optional[datetime] = None + last_error: Optional[str] = None + locked_by: Optional[str] = None + locked_at: Optional[datetime] = None + lease_expires_at: Optional[datetime] = None + metadata: dict[str, Any] = Field(default_factory=dict) + created_at: datetime = Field(default_factory=utc_now) + updated_at: datetime = Field(default_factory=utc_now) + + +class CommitJob(BaseModel): + job_id: str = Field(default_factory=lambda: f"job_{uuid4().hex[:16]}") + workspace_id: str + user_id: str + agent_id: Optional[str] = None + session_id: str + namespace: Optional[str] = None + status: OperationStatus = OperationStatus.ACCEPTED + requested_by: Optional[str] = None + created_refs_count: int = 0 + error_message: Optional[str] = None + created_at: datetime = Field(default_factory=utc_now) + updated_at: datetime = Field(default_factory=utc_now) + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None diff --git a/memory_gateway/backend_normalization.py b/memory_gateway/backend_normalization.py new file mode 100644 index 0000000..a3877a5 --- /dev/null +++ b/memory_gateway/backend_normalization.py @@ -0,0 +1,259 @@ +"""Offline response normalization helpers for future v2 backend adapters.""" +from __future__ import annotations + +from typing import Any + +from .backend_contracts import ( + BackendCommitResult, + BackendOperation, + BackendProducedRef, + BackendResultStatus, + BackendRetrieveItem, + BackendRetrieveResult, + BackendWriteResult, +) +from .backend_ref_mapping import map_backend_ref_type +from .schemas_v2 import BackendType + + +SAFE_METADATA_KEYS = { + "backend_request_id", + "request_id", + "trace_id", + "latency_ms", + "schema_version", + "source_channel", + "reason", + "original_ref_type", + "confidence", + "score", + "version", + "created_at", +} +BLOCKED_METADATA_KEYS = {"content", "raw_request", "messages", "conversation", "transcript"} + + +def safe_backend_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]: + if not metadata: + return {} + safe: dict[str, Any] = {} + for key, value in metadata.items(): + if key in BLOCKED_METADATA_KEYS or key not in SAFE_METADATA_KEYS: + continue + if isinstance(value, (str, int, float, bool)) or value is None: + safe[key] = value + return safe + + +def normalize_openviking_commit_response(raw: dict[str, Any]) -> BackendCommitResult: + status = _result_status(raw) + refs = [_produced_ref(BackendType.OPENVIKING, item) for item in _extract_ref_items(raw)] + return BackendCommitResult( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.COMMIT_SESSION, + status=status, + native_id=raw.get("native_id") or raw.get("session_id"), + native_uri=raw.get("native_uri") or raw.get("uri"), + retryable=_retryable_from_raw(BackendType.OPENVIKING, raw), + error_code=raw.get("error_code"), + error_message=raw.get("error") or raw.get("error_message"), + latency_ms=raw.get("latency_ms"), + refs=refs, + metadata=safe_backend_metadata(raw.get("metadata") or raw), + ) + + +def normalize_evermemos_commit_response(raw: dict[str, Any]) -> BackendCommitResult: + status = _result_status(raw) + refs = [_produced_ref(BackendType.EVERMEMOS, item) for item in _extract_ref_items(raw)] + return BackendCommitResult( + backend_type=BackendType.EVERMEMOS, + operation=BackendOperation.COMMIT_SESSION, + status=status, + native_id=raw.get("native_id") or raw.get("session_id"), + native_uri=raw.get("native_uri") or raw.get("uri"), + retryable=_retryable_from_raw(BackendType.EVERMEMOS, raw), + error_code=raw.get("error_code"), + error_message=raw.get("error") or raw.get("error_message"), + latency_ms=raw.get("latency_ms"), + refs=refs, + metadata=safe_backend_metadata(raw.get("metadata") or raw), + ) + + +def normalize_openviking_ingest_response(raw: dict[str, Any]) -> BackendWriteResult: + return _write_result(BackendType.OPENVIKING, raw) + + +def normalize_evermemos_ingest_response(raw: dict[str, Any]) -> BackendWriteResult: + return _write_result(BackendType.EVERMEMOS, raw) + + +def normalize_openviking_retrieve_response(raw: dict[str, Any]) -> BackendRetrieveResult: + return _retrieve_result(BackendType.OPENVIKING, raw) + + +def normalize_evermemos_retrieve_response(raw: dict[str, Any]) -> BackendRetrieveResult: + return _retrieve_result(BackendType.EVERMEMOS, raw) + + +def map_backend_error_to_retryable( + backend_type: BackendType, + status_code: int | None = None, + error_code: str | None = None, + error_message: str | None = None, +) -> bool: + """Map backend errors into retryable/non-retryable categories. + + Unknown errors default to retryable because adapter contracts are still + unstable and transient backend/API rollout failures are more likely during + integration. + """ + if status_code in {429, 500, 502, 503, 504}: + return True + if status_code in {400, 401, 403, 404, 422}: + return False + text = f"{error_code or ''} {error_message or ''}".lower() + if "timeout" in text or "network_error" in text or "connection" in text: + return True + if "validation" in text or "unauthorized" in text or "forbidden" in text or "not_found" in text: + return False + return True + + +def _write_result(backend_type: BackendType, raw: dict[str, Any]) -> BackendWriteResult: + data = raw.get("result") or raw.get("data") or {} + if not isinstance(data, dict): + data = {} + native_id = ( + raw.get("native_id") + or raw.get("id") + or raw.get("memory_id") + or raw.get("session_id") + or data.get("native_id") + or data.get("id") + or data.get("memory_id") + or data.get("session_id") + ) + native_uri = ( + raw.get("native_uri") + or raw.get("uri") + or raw.get("url") + or data.get("native_uri") + or data.get("uri") + or data.get("url") + ) + if not native_uri and backend_type == BackendType.OPENVIKING and native_id: + native_uri = f"viking://sessions/{native_id}" + return BackendWriteResult( + backend_type=backend_type, + operation=BackendOperation.INGEST_TURN, + status=_result_status(raw), + native_id=native_id, + native_uri=native_uri, + retryable=_retryable_from_raw(backend_type, raw), + error_code=raw.get("error_code"), + error_message=raw.get("error") or raw.get("error_message"), + latency_ms=raw.get("latency_ms"), + metadata=safe_backend_metadata(raw.get("metadata") or raw), + ) + + +def _retrieve_result(backend_type: BackendType, raw: dict[str, Any]) -> BackendRetrieveResult: + if not isinstance(raw, dict) or not raw: + return BackendRetrieveResult( + backend_type=backend_type, + operation=BackendOperation.RETRIEVE_CONTEXT, + status=BackendResultStatus.SKIPPED, + metadata={"reason": "malformed_or_empty_response"}, + ) + return BackendRetrieveResult( + backend_type=backend_type, + operation=BackendOperation.RETRIEVE_CONTEXT, + status=_result_status(raw), + native_id=raw.get("native_id") or raw.get("session_id"), + native_uri=raw.get("native_uri") or raw.get("uri"), + retryable=_retryable_from_raw(backend_type, raw), + error_code=raw.get("error_code"), + error_message=raw.get("error") or raw.get("error_message"), + latency_ms=raw.get("latency_ms"), + items=[_retrieve_item(backend_type, item) for item in _extract_retrieve_items(raw)], + metadata=safe_backend_metadata(raw.get("metadata") or raw), + ) + + +def _retrieve_item(backend_type: BackendType, item: dict[str, Any]) -> BackendRetrieveItem: + metadata = safe_backend_metadata(item.get("metadata") if isinstance(item.get("metadata"), dict) else item) + return BackendRetrieveItem( + text=item.get("text") or item.get("summary") or item.get("abstract"), + source_backend=backend_type, + ref_id=item.get("ref_id") or item.get("id") or item.get("memory_id") or item.get("profile_id") or item.get("session_id") or item.get("uri"), + score=float(item.get("score") or 0.0), + memory_type=item.get("memory_type") or item.get("ref_type") or item.get("type") or item.get("kind"), + metadata=metadata, + ) + + +def _produced_ref(backend_type: BackendType, item: dict[str, Any]) -> BackendProducedRef: + ref_type, mapping_metadata = map_backend_ref_type(backend_type, item.get("ref_type") or item.get("type") or item.get("kind")) + metadata = { + **safe_backend_metadata(item.get("metadata") if isinstance(item.get("metadata"), dict) else item), + **mapping_metadata, + } + return BackendProducedRef( + ref_type=ref_type, + native_id=item.get("native_id") or item.get("id") or item.get("memory_id") or item.get("profile_id") or item.get("session_id"), + native_uri=item.get("native_uri") or item.get("uri") or item.get("url"), + metadata=metadata, + ) + + +def _extract_ref_items(raw: dict[str, Any]) -> list[dict[str, Any]]: + data = raw.get("result") or raw.get("data") or raw + candidates = ( + data.get("refs") + or data.get("produced_refs") + or data.get("created_refs") + or data.get("memories") + or data.get("items") + or [] + ) + return [item for item in candidates if isinstance(item, dict)] + + +def _extract_retrieve_items(raw: dict[str, Any]) -> list[dict[str, Any]]: + data = raw.get("result") or raw.get("data") or raw + if not isinstance(data, dict): + return [] + candidates = ( + data.get("items") + or data.get("results") + or data.get("memories") + or data.get("resources") + or data.get("contexts") + or [] + ) + return [item for item in candidates if isinstance(item, dict)] + + +def _result_status(raw: dict[str, Any]) -> BackendResultStatus: + status = str(raw.get("status") or "success").lower() + if status in {"ok", "created", "accepted"}: + return BackendResultStatus.SUCCESS + try: + return BackendResultStatus(status) + except ValueError: + return BackendResultStatus.SUCCESS if not raw.get("error") and not raw.get("error_message") else BackendResultStatus.FAILED + + +def _retryable_from_raw(backend_type: BackendType, raw: dict[str, Any]) -> bool: + if "retryable" in raw: + return bool(raw["retryable"]) + if raw.get("error") or raw.get("error_message") or raw.get("error_code") or raw.get("status_code"): + return map_backend_error_to_retryable( + backend_type, + status_code=raw.get("status_code"), + error_code=raw.get("error_code"), + error_message=raw.get("error") or raw.get("error_message"), + ) + return False diff --git a/memory_gateway/backend_ref_mapping.py b/memory_gateway/backend_ref_mapping.py new file mode 100644 index 0000000..3212f94 --- /dev/null +++ b/memory_gateway/backend_ref_mapping.py @@ -0,0 +1,66 @@ +"""Backend-specific ref type mapping for Memory Gateway v2.""" +from __future__ import annotations + +from .schemas_v2 import BackendType, MemoryRefType + + +OPENVIKING_REF_TYPE_MAP = { + "session_archive": MemoryRefType.SESSION_ARCHIVE, + "context_resource": MemoryRefType.CONTEXT_RESOURCE, + "resource": MemoryRefType.CONTEXT_RESOURCE, + "session_summary": MemoryRefType.SESSION_ARCHIVE, +} + +EVERMEMOS_REF_TYPE_MAP = { + "message_memory": MemoryRefType.MESSAGE_MEMORY, + "episodic_memory": MemoryRefType.EPISODIC_MEMORY, + "episode": MemoryRefType.EPISODIC_MEMORY, + "profile": MemoryRefType.PROFILE, + "long_term_memory": MemoryRefType.LONG_TERM_MEMORY, + "memory": MemoryRefType.LONG_TERM_MEMORY, + "preference": MemoryRefType.PROFILE, +} + +OBSIDIAN_REF_TYPE_MAP = { + "review_draft": MemoryRefType.DRAFT_REVIEW, + "draft_review": MemoryRefType.DRAFT_REVIEW, +} + + +def map_backend_ref_type( + backend_type: BackendType, + backend_ref_type: str | None, +) -> tuple[MemoryRefType, dict[str, str]]: + """Map backend-native ref type to a Gateway MemoryRefType. + + Unknown values fall back to a backend-appropriate default and preserve the + original value in returned metadata for later inspection. + """ + raw_type = (backend_ref_type or "").strip() + normalized = raw_type.lower() + + if backend_type == BackendType.OPENVIKING: + mapped = OPENVIKING_REF_TYPE_MAP.get(normalized, MemoryRefType.SESSION_ARCHIVE) + elif backend_type == BackendType.EVERMEMOS: + mapped = EVERMEMOS_REF_TYPE_MAP.get(normalized, MemoryRefType.LONG_TERM_MEMORY) + elif backend_type == BackendType.OBSIDIAN: + mapped = OBSIDIAN_REF_TYPE_MAP.get(normalized, MemoryRefType.DRAFT_REVIEW) + else: + mapped = MemoryRefType.LONG_TERM_MEMORY + + metadata: dict[str, str] = {} + if raw_type and raw_type not in {mapped.value, normalized}: + metadata["original_ref_type"] = raw_type + elif raw_type and normalized not in _known_backend_ref_types(backend_type): + metadata["original_ref_type"] = raw_type + return mapped, metadata + + +def _known_backend_ref_types(backend_type: BackendType) -> set[str]: + if backend_type == BackendType.OPENVIKING: + return set(OPENVIKING_REF_TYPE_MAP) + if backend_type == BackendType.EVERMEMOS: + return set(EVERMEMOS_REF_TYPE_MAP) + if backend_type == BackendType.OBSIDIAN: + return set(OBSIDIAN_REF_TYPE_MAP) + return set() diff --git a/memory_gateway/config.py b/memory_gateway/config.py index 1ca243a..852184a 100644 --- a/memory_gateway/config.py +++ b/memory_gateway/config.py @@ -18,16 +18,16 @@ def load_config(config_path: Optional[str] = None) -> Config: if not config_file.exists(): # 返回默认配置 - return Config() + return _apply_env_overrides(Config()) try: with open(config_file, "r", encoding="utf-8") as f: data = yaml.safe_load(f) if data is None: - return Config() + return _apply_env_overrides(Config()) - return Config( + config = Config( server=ServerConfig(**data.get("server", {})), openviking=OpenVikingConfig(**data.get("openviking", {})), evermemos=EverMemOSConfig(**data.get("evermemos", {})), @@ -37,9 +37,10 @@ def load_config(config_path: Optional[str] = None) -> Config: obsidian=ObsidianConfig(**data.get("obsidian", {})), storage=StorageConfig(**data.get("storage", {})), ) + return _apply_env_overrides(config) except (ValidationError, yaml.YAMLError) as e: print(f"配置文件解析错误: {e}") - return Config() + return _apply_env_overrides(Config()) def get_config() -> Config: @@ -57,3 +58,42 @@ def set_config(config: Config) -> None: _config: Optional[Config] = None + + +def _apply_env_overrides(config: Config) -> Config: + openviking_updates = _backend_env_updates("OPENVIKING") + evermemos_updates = _backend_env_updates("EVERMEMOS") + if openviking_updates: + config.openviking = config.openviking.model_copy(update=openviking_updates) + if evermemos_updates: + config.evermemos = config.evermemos.model_copy(update=evermemos_updates) + return config + + +def _backend_env_updates(prefix: str) -> dict: + updates = {} + env_map = { + "ENABLED": "enabled", + "MODE": "mode", + "BASE_URL": "url", + "URL": "url", + "API_KEY": "api_key", + "TOKEN": "api_key", + "TIMEOUT": "timeout", + "TIMEOUT_SECONDS": "timeout", + "VERIFY_SSL": "verify_ssl", + "INGEST_PATH": "ingest_path", + } + for env_name, field_name in env_map.items(): + value = os.environ.get(f"{prefix}_{env_name}") + if value is None: + continue + if field_name == "enabled": + updates[field_name] = value.lower() in {"1", "true", "yes", "on"} + elif field_name == "timeout": + updates[field_name] = int(value) + elif field_name == "verify_ssl": + updates[field_name] = value.lower() not in {"0", "false", "no", "off"} + else: + updates[field_name] = value + return updates diff --git a/memory_gateway/evermemos_client.py b/memory_gateway/evermemos_client.py index 169fbe6..c0b1b7f 100644 --- a/memory_gateway/evermemos_client.py +++ b/memory_gateway/evermemos_client.py @@ -1,12 +1,21 @@ """Client for the external EverMemOS consolidation service.""" from __future__ import annotations +from json import JSONDecodeError from typing import Any import httpx +from .backend_contracts import BackendCommitResult, BackendOperation, BackendResultStatus, BackendRetrieveResult, BackendWriteResult +from .backend_normalization import ( + map_backend_error_to_retryable, + normalize_evermemos_commit_response, + normalize_evermemos_ingest_response, + normalize_evermemos_retrieve_response, +) from .config import get_config from .schemas import AccessContext, EpisodeRecord, MemoryRecord +from .schemas_v2 import BackendType class EverMemOSError(RuntimeError): @@ -26,15 +35,25 @@ class EverMemOSClient: base_url: str | None = None, api_key: str | None = None, timeout: int | None = None, + enabled: bool | None = None, + mode: str | None = None, + verify_ssl: bool | None = None, health_path: str | None = None, + ingest_path: str | None = None, consolidate_path: str | None = None, + transport: httpx.BaseTransport | None = None, ) -> None: config = get_config().evermemos - self.base_url = (base_url or config.url).rstrip("/") + self.base_url = (base_url if base_url is not None else config.url).rstrip("/") self.api_key = api_key if api_key is not None else config.api_key self.timeout = timeout or config.timeout + self.enabled = config.enabled if enabled is None else enabled + self.mode = mode or config.mode + self.verify_ssl = config.verify_ssl if verify_ssl is None else verify_ssl self.health_path = health_path or config.health_path + self.ingest_path = ingest_path or config.ingest_path self.consolidate_path = consolidate_path or config.consolidate_path + self.transport = transport def _headers(self) -> dict[str, str]: headers = {"Content-Type": "application/json"} @@ -46,13 +65,195 @@ class EverMemOSClient: def health(self) -> dict[str, Any]: url = self.base_url + self.health_path try: - with httpx.Client(timeout=self.timeout, headers=self._headers()) as client: + health_timeout = httpx.Timeout(min(self.timeout, 2.0), connect=min(self.timeout, 0.5)) + with httpx.Client(timeout=health_timeout, headers=self._headers()) as client: response = client.get(url) response.raise_for_status() return {"status": "ok", "url": self.base_url, "response": response.json()} except Exception as exc: # noqa: BLE001 return {"status": "error", "url": self.base_url, "error": str(exc)} + def ingest_message(self, payload: dict[str, Any]) -> BackendWriteResult: + """v2 adapter placeholder for message-level EverMemOS ingestion. + + Mapping spec: `backend_adapter_mapping.AdapterMappingSpec` maps + EverMemOS ingest_turn to this method and requires BackendWriteResult. + Payloads must contain only control-plane fields; raw request bodies are + not persisted by the Gateway control-plane store. + + TODO(v2): bind this to EverMemOS `/api/v1/memories` or its stable + message ingestion API after the external contract settles. + """ + runtime_payload = self._build_ingest_payload(payload) + if self._use_real_api: + return self._ingest_message_real(runtime_payload) + raw = { + "status": "skipped", + "memory_id": runtime_payload.get("turn_id"), + "metadata": { + "reason": "evermemos_v2_ingest_adapter_not_configured", + "schema_version": "evermemos.fixture.ingest.v2", + }, + } + return self._normalize_ingest_response(raw) + + @property + def _use_real_api(self) -> bool: + # Real ingest is strictly gated by mode=real. The legacy `enabled` + # field is retained for config compatibility, but must not trigger + # network traffic by itself. + return self.mode == "real" + + def _ingest_message_real(self, runtime_payload: dict[str, Any]) -> BackendWriteResult: + if not self.base_url: + return self._failed_ingest_result( + error_code="config_error", + error_message="EverMemOS real ingest is enabled but base_url is missing", + retryable=False, + ) + try: + with httpx.Client( + base_url=self.base_url, + headers=self._headers(), + timeout=self.timeout, + verify=self.verify_ssl, + transport=self.transport, + ) as client: + response = client.post(self.ingest_path, json=runtime_payload) + if response.status_code >= 400: + return self._failed_ingest_result( + error_code=f"http_{response.status_code}", + error_message=f"EverMemOS ingest failed with HTTP {response.status_code}", + retryable=self._map_error(response), + ) + try: + raw = response.json() + except (JSONDecodeError, ValueError): + return self._failed_ingest_result( + error_code="invalid_json", + error_message="EverMemOS ingest returned invalid JSON", + retryable=True, + ) + if not isinstance(raw, dict): + return self._failed_ingest_result( + error_code="unexpected_response", + error_message="EverMemOS ingest returned an unexpected response shape", + retryable=True, + ) + return self._normalize_ingest_response(raw) + except httpx.TimeoutException as exc: + return self._failed_ingest_result("timeout", self._safe_error_message(exc), retryable=self._map_error(exc)) + except httpx.RequestError as exc: + return self._failed_ingest_result("network_error", self._safe_error_message(exc), retryable=self._map_error(exc)) + except Exception as exc: # noqa: BLE001 + return self._failed_ingest_result("unexpected_error", self._safe_error_message(exc), retryable=self._map_error(exc)) + + def extract_profile_long_term_v2(self, payload: dict[str, Any]) -> BackendCommitResult: + """v2 adapter placeholder for profile / long-term extraction. + + Mapping spec: commit_session returns BackendCommitResult and should + produce native episodic/profile/long-term refs once the real API is stable. + """ + runtime_payload = self._build_commit_payload(payload) + raw = { + "status": "success", + "session_id": runtime_payload.get("session_id"), + "metadata": { + "reason": "evermemos_v2_commit_fixture", + "schema_version": "evermemos.fixture.commit.v2", + }, + "data": { + "produced_refs": [ + { + "ref_type": "profile", + "profile_id": f"em_profile:{runtime_payload.get('user_id') or 'unknown'}", + "metadata": {"schema_version": "evermemos.fixture.profile.v2"}, + }, + { + "ref_type": "long_term_memory", + "memory_id": f"em_long_term:{runtime_payload.get('session_id')}", + "metadata": {"schema_version": "evermemos.fixture.long_term.v2"}, + }, + ] + }, + } + return self._normalize_commit_response(raw) + + def retrieve_context_v2(self, payload: dict[str, Any]) -> BackendRetrieveResult: + """v2 adapter placeholder for episodic/profile/long-term retrieval. + + Mapping spec: retrieve_context returns BackendRetrieveResult with + normalized context items, not raw backend payload dumps. + """ + raw = { + "status": "success", + "metadata": { + "reason": "evermemos_v2_retrieve_fixture", + "schema_version": "evermemos.fixture.retrieve.v2", + }, + "data": { + "items": [ + { + "text": "EverMemOS fixture profile context.", + "profile_id": f"em_profile:{payload.get('user_id') or 'unknown'}", + "score": 0.72, + "memory_type": "profile", + "metadata": {"schema_version": "evermemos.fixture.retrieve.item.v2"}, + }, + { + "text": "EverMemOS fixture long-term memory context.", + "memory_id": f"em_long_term:{payload.get('session_id') or 'unknown'}", + "score": 0.69, + "memory_type": "long_term_memory", + "metadata": {"schema_version": "evermemos.fixture.retrieve.item.v2"}, + }, + ] + }, + } + return self._normalize_retrieve_response(raw) + + def _build_ingest_payload(self, payload: dict[str, Any]) -> dict[str, Any]: + # Runtime-only adapter payload. It may include conversation content for + # the current request lifecycle; callers must not persist it to SQLite. + return dict(payload) + + def _build_commit_payload(self, payload: dict[str, Any]) -> dict[str, Any]: + return dict(payload) + + def _normalize_ingest_response(self, raw: dict[str, Any]) -> BackendWriteResult: + return normalize_evermemos_ingest_response(raw) + + def _normalize_commit_response(self, raw: dict[str, Any]) -> BackendCommitResult: + return normalize_evermemos_commit_response(raw) + + def _normalize_retrieve_response(self, raw: dict[str, Any]) -> BackendRetrieveResult: + return normalize_evermemos_retrieve_response(raw) + + def _map_error(self, exc_or_response: Any) -> bool: + status_code = getattr(exc_or_response, "status_code", None) + error_code = getattr(exc_or_response, "error_code", None) + error_message = str(exc_or_response) if exc_or_response is not None else None + return map_backend_error_to_retryable( + BackendType.EVERMEMOS, + status_code=status_code, + error_code=error_code, + error_message=error_message, + ) + + def _failed_ingest_result(self, error_code: str, error_message: str, retryable: bool) -> BackendWriteResult: + return BackendWriteResult( + backend_type=BackendType.EVERMEMOS, + operation=BackendOperation.INGEST_TURN, + status=BackendResultStatus.FAILED, + retryable=retryable, + error_code=error_code, + error_message=error_message, + metadata={"error_code": error_code}, + ) + + def _safe_error_message(self, exc: Exception) -> str: + return exc.__class__.__name__ + def consolidate_session( self, session_id: str, @@ -110,4 +311,3 @@ class EverMemOSClient: "conflicts": data.get("conflicts") or [], "review_drafts": data.get("review_drafts") or [], } - diff --git a/memory_gateway/evermemos_service.py b/memory_gateway/evermemos_service.py index 840ccad..b0ac058 100644 --- a/memory_gateway/evermemos_service.py +++ b/memory_gateway/evermemos_service.py @@ -13,6 +13,7 @@ conflicting candidates. from __future__ import annotations import argparse +import hashlib import logging from typing import Any @@ -37,6 +38,18 @@ class ConsolidateRequest(BaseModel): existing_memories: list[dict[str, Any]] = Field(default_factory=list) +class MemoryIngestRequest(BaseModel): + workspace_id: str | None = None + user_id: str + session_id: str + turn_id: str + role: str = "user" + content: str + metadata: dict[str, Any] = Field(default_factory=dict) + source_type: str | None = None + source_event_id: str | None = None + + app = FastAPI(title="Local EverMemOS POC Service", version="0.1.0") @@ -50,6 +63,35 @@ async def health() -> dict[str, Any]: } +@app.post("/api/v1/memories") +async def ingest_memory(request: MemoryIngestRequest) -> dict[str, Any]: + """Accept message-level ingest for local real-adapter smoke tests. + + This POC endpoint intentionally does not persist raw conversation content. + It only returns a stable backend reference that Memory Gateway can store as + control-plane metadata. + """ + seed = "|".join( + [ + request.workspace_id or "", + request.user_id, + request.session_id, + request.turn_id, + request.source_event_id or "", + ] + ) + memory_id = "em_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] + return { + "status": "success", + "memory_id": memory_id, + "native_uri": f"evermemos://memories/{memory_id}", + "metadata": { + "schema_version": "evermemos.local.ingest.v1", + "source_channel": request.metadata.get("source_channel") or request.metadata.get("channel"), + }, + } + + @app.post("/v1/sessions/consolidate") async def consolidate_session(request: ConsolidateRequest) -> dict[str, Any]: repo = InMemoryRepository() @@ -105,4 +147,3 @@ def main() -> None: if __name__ == "__main__": main() - diff --git a/memory_gateway/obsidian_review_client.py b/memory_gateway/obsidian_review_client.py new file mode 100644 index 0000000..aa9f366 --- /dev/null +++ b/memory_gateway/obsidian_review_client.py @@ -0,0 +1,25 @@ +"""Human-review backend skeleton for Memory Gateway v2. + +Obsidian remains a human-in-the-loop review backend only. This skeleton does +not write files or call external APIs; it preserves the adapter contract until +the review draft integration is explicitly designed. +""" +from __future__ import annotations + +from typing import Any + +from .backend_contracts import BackendOperation, BackendResultStatus, BackendWriteResult +from .schemas_v2 import BackendType + + +class ObsidianReviewClient: + def create_review_draft_v2(self, payload: dict[str, Any]) -> BackendWriteResult: + """Return a skipped review-draft result until the real adapter exists.""" + return BackendWriteResult( + backend_type=BackendType.OBSIDIAN, + operation=BackendOperation.CREATE_REVIEW_DRAFT, + status=BackendResultStatus.SKIPPED, + native_id=payload.get("event_id") or payload.get("gateway_id"), + retryable=False, + metadata={"reason": "obsidian_review_adapter_not_configured"}, + ) diff --git a/memory_gateway/openviking_client.py b/memory_gateway/openviking_client.py index c97d7ed..e3de4dd 100644 --- a/memory_gateway/openviking_client.py +++ b/memory_gateway/openviking_client.py @@ -5,12 +5,21 @@ import json import logging import mimetypes import tempfile +from json import JSONDecodeError from pathlib import Path from typing import Any, Optional import httpx +from .backend_contracts import BackendCommitResult, BackendOperation, BackendResultStatus, BackendRetrieveResult, BackendWriteResult +from .backend_normalization import ( + map_backend_error_to_retryable, + normalize_openviking_commit_response, + normalize_openviking_ingest_response, + normalize_openviking_retrieve_response, +) from .config import get_config +from .schemas_v2 import BackendType from .types import MemoryEntry, ResourceEntry, SearchResult logger = logging.getLogger(__name__) @@ -23,16 +32,26 @@ class OpenVikingClient: self, base_url: Optional[str] = None, api_key: Optional[str] = None, - timeout: int = 30, + timeout: int | None = None, account: str = "default", user: str = "default", + enabled: bool | None = None, + mode: str | None = None, + verify_ssl: bool | None = None, + ingest_path: str | None = None, + transport: httpx.AsyncBaseTransport | None = None, ): self.config = get_config() - self.base_url = base_url or self.config.openviking.url - self.api_key = api_key or self.config.openviking.api_key or "your-secret-root-key" - self.timeout = timeout + self.base_url = base_url if base_url is not None else self.config.openviking.url + self.api_key = api_key if api_key is not None else (self.config.openviking.api_key or "your-secret-root-key") + self.timeout = timeout if timeout is not None else self.config.openviking.timeout self.account = account self.user = user + self.enabled = self.config.openviking.enabled if enabled is None else enabled + self.mode = mode or self.config.openviking.mode + self.verify_ssl = self.config.openviking.verify_ssl if verify_ssl is None else verify_ssl + self.ingest_path = ingest_path or self.config.openviking.ingest_path + self.transport = transport self._client: Optional[httpx.AsyncClient] = None def _get_headers(self) -> dict[str, str]: @@ -49,6 +68,8 @@ class OpenVikingClient: base_url=self.base_url, headers=self._get_headers(), timeout=self.timeout, + verify=self.verify_ssl, + transport=self.transport, ) return self._client @@ -67,6 +88,180 @@ class OpenVikingClient: logger.error(f"OpenViking 健康检查失败: {e}") return {"status": "error", "message": str(e)} + async def ingest_conversation_turn(self, payload: dict[str, Any]) -> BackendWriteResult: + """v2 adapter placeholder for OpenViking session archive ingestion. + + Mapping spec: `backend_adapter_mapping.AdapterMappingSpec` maps + OpenViking ingest_turn to this method and requires BackendWriteResult. + Payloads must contain only control-plane fields; conversation content + is not persisted by the Gateway control-plane store. + + TODO(v2): bind this to OpenViking's stable session/message archive API + once that contract is finalized. Until then the gateway records a + skipped backend ref instead of inventing an unstable HTTP contract. + """ + runtime_payload = self._build_ingest_payload(payload) + if self._use_real_api: + return await self._ingest_conversation_turn_real(runtime_payload) + raw = { + "status": "skipped", + "session_id": runtime_payload.get("session_id"), + "uri": f"viking://sessions/{runtime_payload.get('session_id')}", + "metadata": { + "reason": "openviking_v2_ingest_adapter_not_configured", + "schema_version": "openviking.fixture.ingest.v2", + }, + } + return self._normalize_ingest_response(raw) + + @property + def _use_real_api(self) -> bool: + # Real ingest is strictly gated by mode=real. The legacy `enabled` + # field is retained for config compatibility, but must not trigger + # network traffic by itself. + return self.mode == "real" + + async def _ingest_conversation_turn_real(self, runtime_payload: dict[str, Any]) -> BackendWriteResult: + if not self.base_url: + return self._failed_ingest_result( + error_code="config_error", + error_message="OpenViking real ingest is enabled but base_url is missing", + retryable=False, + ) + try: + client = await self._get_client() + response = await client.post( + self._format_ingest_path(runtime_payload), + json=runtime_payload, + ) + if response.status_code >= 400: + return self._failed_ingest_result( + error_code=f"http_{response.status_code}", + error_message=f"OpenViking ingest failed with HTTP {response.status_code}", + retryable=self._map_error(response), + ) + try: + raw = response.json() + except (JSONDecodeError, ValueError): + return self._failed_ingest_result( + error_code="invalid_json", + error_message="OpenViking ingest returned invalid JSON", + retryable=True, + ) + if not isinstance(raw, dict): + return self._failed_ingest_result( + error_code="unexpected_response", + error_message="OpenViking ingest returned an unexpected response shape", + retryable=True, + ) + return self._normalize_ingest_response(raw) + except httpx.TimeoutException as exc: + return self._failed_ingest_result("timeout", self._safe_error_message(exc), retryable=self._map_error(exc)) + except httpx.RequestError as exc: + return self._failed_ingest_result("network_error", self._safe_error_message(exc), retryable=self._map_error(exc)) + except Exception as exc: # noqa: BLE001 + return self._failed_ingest_result("unexpected_error", self._safe_error_message(exc), retryable=self._map_error(exc)) + + async def commit_session_v2(self, payload: dict[str, Any]) -> BackendCommitResult: + """v2 adapter placeholder for OpenViking session commit. + + Mapping spec: commit_session returns BackendCommitResult and should + produce a native session/archive ref once the real API is stable. + """ + runtime_payload = self._build_commit_payload(payload) + raw = { + "status": "success", + "session_id": runtime_payload.get("session_id"), + "metadata": { + "reason": "openviking_v2_commit_fixture", + "schema_version": "openviking.fixture.commit.v2", + }, + "result": { + "refs": [ + { + "type": "session_summary", + "id": f"ov_session_summary:{runtime_payload.get('session_id')}", + "uri": f"viking://sessions/{runtime_payload.get('session_id')}/summary", + "metadata": {"schema_version": "openviking.fixture.ref.v2"}, + } + ] + }, + } + return self._normalize_commit_response(raw) + + async def retrieve_context_v2(self, payload: dict[str, Any]) -> BackendRetrieveResult: + """v2 adapter placeholder for OpenViking runtime context retrieval. + + Mapping spec: retrieve_context returns BackendRetrieveResult with + runtime context items, not raw backend payload dumps. + """ + raw = { + "status": "ok", + "session_id": payload.get("session_id"), + "metadata": { + "reason": "openviking_v2_retrieve_fixture", + "schema_version": "openviking.fixture.retrieve.v2", + }, + "result": { + "items": [ + { + "text": "OpenViking fixture runtime context.", + "ref_id": f"ov_context:{payload.get('session_id') or 'unknown'}", + "score": 0.75, + "memory_type": "context_resource", + "metadata": {"schema_version": "openviking.fixture.retrieve.item.v2"}, + } + ] + }, + } + return self._normalize_retrieve_response(raw) + + def _build_ingest_payload(self, payload: dict[str, Any]) -> dict[str, Any]: + # Runtime-only adapter payload. It may include conversation content for + # the current request lifecycle; callers must not persist it to SQLite. + return dict(payload) + + def _format_ingest_path(self, payload: dict[str, Any]) -> str: + session_id = str(payload.get("session_id") or "unknown") + return self.ingest_path.format(session_id=session_id) + + def _build_commit_payload(self, payload: dict[str, Any]) -> dict[str, Any]: + return dict(payload) + + def _normalize_ingest_response(self, raw: dict[str, Any]) -> BackendWriteResult: + return normalize_openviking_ingest_response(raw) + + def _normalize_commit_response(self, raw: dict[str, Any]) -> BackendCommitResult: + return normalize_openviking_commit_response(raw) + + def _normalize_retrieve_response(self, raw: dict[str, Any]) -> BackendRetrieveResult: + return normalize_openviking_retrieve_response(raw) + + def _map_error(self, exc_or_response: Any) -> bool: + status_code = getattr(exc_or_response, "status_code", None) + error_code = getattr(exc_or_response, "error_code", None) + error_message = str(exc_or_response) if exc_or_response is not None else None + return map_backend_error_to_retryable( + BackendType.OPENVIKING, + status_code=status_code, + error_code=error_code, + error_message=error_message, + ) + + def _failed_ingest_result(self, error_code: str, error_message: str, retryable: bool) -> BackendWriteResult: + return BackendWriteResult( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.INGEST_TURN, + status=BackendResultStatus.FAILED, + retryable=retryable, + error_code=error_code, + error_message=error_message, + metadata={"error_code": error_code}, + ) + + def _safe_error_message(self, exc: Exception) -> str: + return exc.__class__.__name__ + async def search( self, query: str, diff --git a/memory_gateway/repositories.py b/memory_gateway/repositories.py index ac9c71b..a6e0fa1 100644 --- a/memory_gateway/repositories.py +++ b/memory_gateway/repositories.py @@ -7,12 +7,14 @@ from __future__ import annotations import json import sqlite3 -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Iterable, Optional, Protocol +from .backend_contracts import BackendOperation, CommitJob, OutboxEvent, OutboxEventStatus from .config import get_config from .schemas import AuditLog, EpisodeRecord, MemoryRecord, ProfileRecord, UserRecord +from .schemas_v2 import BackendRefStatus, BackendType, MemoryRef, MemoryRefType class MetadataRepository(Protocol): @@ -28,6 +30,61 @@ class MetadataRepository(Protocol): def upsert_profile(self, profile: ProfileRecord) -> ProfileRecord: ... def add_audit(self, audit: AuditLog) -> AuditLog: ... def list_audit(self, limit: int = 100) -> list[AuditLog]: ... + def save_memory_ref(self, ref: MemoryRef) -> MemoryRef: ... + def get_memory_ref(self, ref_id: str) -> MemoryRef | None: ... + def list_memory_refs( + self, + gateway_id: str | None = None, + workspace_id: str | None = None, + user_id: str | None = None, + agent_id: str | None = None, + session_id: str | None = None, + namespace: str | None = None, + backend_type: BackendType | str | None = None, + ref_type: MemoryRefType | str | None = None, + status: BackendRefStatus | str | None = None, + limit: int = 100, + ) -> list[MemoryRef]: ... + def save_outbox_event(self, event: OutboxEvent) -> OutboxEvent: ... + def list_outbox_events( + self, + status: OutboxEventStatus | str | None = None, + backend_type: BackendType | str | None = None, + operation: BackendOperation | str | None = None, + gateway_id: str | None = None, + payload_ref: str | None = None, + limit: int = 100, + ) -> list[OutboxEvent]: ... + def list_outbox_events_by_job(self, job_id: str, limit: int = 100) -> list[OutboxEvent]: ... + def claim_outbox_event(self, event_id: str) -> OutboxEvent | None: ... + def claim_pending_outbox_events( + self, + limit: int, + worker_id: str, + lease_seconds: int, + ) -> list[OutboxEvent]: ... + def release_expired_processing_events(self, now: datetime | None = None) -> list[OutboxEvent]: ... + def update_outbox_event_status( + self, + event_id: str, + status: OutboxEventStatus | str, + last_error: str | None = None, + ) -> OutboxEvent | None: ... + def save_commit_job(self, job: CommitJob) -> CommitJob: ... + def get_commit_job(self, job_id: str) -> CommitJob | None: ... + def update_commit_job_status( + self, + job_id: str, + status: str, + error_message: str | None = None, + created_refs_count: int | None = None, + ) -> CommitJob | None: ... + def count_memory_refs( + self, + gateway_id: str | None = None, + session_id: str | None = None, + status: BackendRefStatus | str | None = None, + ) -> int: ... def _json_dump_model(model) -> str: @@ -38,6 +95,14 @@ def _json_load_model(model_cls, payload: str): return model_cls.model_validate(json.loads(payload)) +def _enum_value(value): + return value.value if hasattr(value, "value") else value + + +def _safe_timedelta(seconds: int) -> timedelta: + return timedelta(seconds=max(1, int(seconds))) + + class InMemoryRepository: def __init__(self) -> None: self.users: dict[str, UserRecord] = {} @@ -45,6 +110,9 @@ class InMemoryRepository: self.episodes: dict[str, EpisodeRecord] = {} self.profiles: dict[str, ProfileRecord] = {} self.audit_logs: list[AuditLog] = [] + self.memory_refs: dict[str, MemoryRef] = {} + self.outbox_events: dict[str, OutboxEvent] = {} + self.commit_jobs: dict[str, CommitJob] = {} def create_user(self, user: UserRecord) -> UserRecord: now = datetime.now(timezone.utc) @@ -102,6 +170,210 @@ class InMemoryRepository: def list_audit(self, limit: int = 100) -> list[AuditLog]: return self.audit_logs[-limit:] + def save_memory_ref(self, ref: MemoryRef) -> MemoryRef: + now = datetime.now(timezone.utc) + existing = self.memory_refs.get(ref.id) + if existing: + ref.created_at = existing.created_at + ref.updated_at = now + self.memory_refs[ref.id] = ref + return ref + + def get_memory_ref(self, ref_id: str) -> MemoryRef | None: + return self.memory_refs.get(ref_id) + + def list_memory_refs( + self, + gateway_id: str | None = None, + workspace_id: str | None = None, + user_id: str | None = None, + agent_id: str | None = None, + session_id: str | None = None, + namespace: str | None = None, + backend_type: BackendType | str | None = None, + ref_type: MemoryRefType | str | None = None, + status: BackendRefStatus | str | None = None, + limit: int = 100, + ) -> list[MemoryRef]: + refs = list(self.memory_refs.values()) + + def matches(ref: MemoryRef) -> bool: + return ( + (workspace_id is None or ref.workspace_id == workspace_id) + and (gateway_id is None or ref.gateway_id == gateway_id) + and (user_id is None or ref.user_id == user_id) + and (agent_id is None or ref.agent_id == agent_id) + and (session_id is None or ref.session_id == session_id) + and (namespace is None or ref.namespace == namespace) + and (backend_type is None or ref.backend_type.value == _enum_value(backend_type)) + and (ref_type is None or ref.ref_type.value == _enum_value(ref_type)) + and (status is None or ref.status.value == _enum_value(status)) + ) + + refs = [ref for ref in refs if matches(ref)] + refs.sort(key=lambda ref: ref.created_at, reverse=True) + return refs[:limit] + + def save_outbox_event(self, event: OutboxEvent) -> OutboxEvent: + now = datetime.now(timezone.utc) + existing = self.outbox_events.get(event.id) + if existing: + event.created_at = existing.created_at + event.updated_at = now + self.outbox_events[event.id] = event + return event + + def list_outbox_events( + self, + status: OutboxEventStatus | str | None = None, + backend_type: BackendType | str | None = None, + operation: BackendOperation | str | None = None, + gateway_id: str | None = None, + payload_ref: str | None = None, + limit: int = 100, + ) -> list[OutboxEvent]: + events = list(self.outbox_events.values()) + + def matches(event: OutboxEvent) -> bool: + return ( + (status is None or event.status.value == _enum_value(status)) + and (backend_type is None or event.backend_type.value == _enum_value(backend_type)) + and (operation is None or event.operation.value == _enum_value(operation)) + and (gateway_id is None or event.gateway_id == gateway_id) + and (payload_ref is None or event.payload_ref == payload_ref) + ) + + events = [event for event in events if matches(event)] + events.sort(key=lambda event: event.created_at, reverse=True) + return events[:limit] + + def list_outbox_events_by_job(self, job_id: str, limit: int = 100) -> list[OutboxEvent]: + return self.list_outbox_events(payload_ref=f"commit_job:{job_id}", limit=limit) + + def claim_outbox_event(self, event_id: str) -> OutboxEvent | None: + event = self.outbox_events.get(event_id) + now = datetime.now(timezone.utc) + if not event or event.status != OutboxEventStatus.PENDING: + return None + if event.next_retry_at and event.next_retry_at > now: + return None + event.status = OutboxEventStatus.PROCESSING + event.locked_by = "inline" + event.locked_at = now + event.lease_expires_at = now + _safe_timedelta(300) + event.updated_at = now + self.outbox_events[event.id] = event + return event + + def claim_pending_outbox_events( + self, + limit: int, + worker_id: str, + lease_seconds: int, + ) -> list[OutboxEvent]: + now = datetime.now(timezone.utc) + candidates = [ + event + for event in self.outbox_events.values() + if event.status == OutboxEventStatus.PENDING + and (event.next_retry_at is None or event.next_retry_at <= now) + ] + candidates.sort(key=lambda event: event.created_at) + claimed: list[OutboxEvent] = [] + for event in candidates[:limit]: + event.status = OutboxEventStatus.PROCESSING + event.locked_by = worker_id + event.locked_at = now + event.lease_expires_at = now + _safe_timedelta(lease_seconds) + event.updated_at = now + self.outbox_events[event.id] = event + claimed.append(event) + return claimed + + def release_expired_processing_events(self, now: datetime | None = None) -> list[OutboxEvent]: + now = now or datetime.now(timezone.utc) + released: list[OutboxEvent] = [] + for event in list(self.outbox_events.values()): + if ( + event.status == OutboxEventStatus.PROCESSING + and event.lease_expires_at is not None + and event.lease_expires_at <= now + ): + event.status = OutboxEventStatus.PENDING + event.locked_by = None + event.locked_at = None + event.lease_expires_at = None + event.updated_at = now + self.outbox_events[event.id] = event + released.append(event) + return released + + def update_outbox_event_status( + self, + event_id: str, + status: OutboxEventStatus | str, + last_error: str | None = None, + ) -> OutboxEvent | None: + event = self.outbox_events.get(event_id) + if not event: + return None + event.status = OutboxEventStatus(_enum_value(status)) + event.last_error = last_error + event.updated_at = datetime.now(timezone.utc) + if event.status != OutboxEventStatus.PROCESSING: + event.locked_by = None + event.locked_at = None + event.lease_expires_at = None + if event.status in {OutboxEventStatus.FAILED, OutboxEventStatus.DEAD_LETTER}: + event.attempt_count += 1 + self.outbox_events[event.id] = event + return event + + def save_commit_job(self, job: CommitJob) -> CommitJob: + now = datetime.now(timezone.utc) + existing = self.commit_jobs.get(job.job_id) + if existing: + job.created_at = existing.created_at + job.updated_at = now + self.commit_jobs[job.job_id] = job + return job + + def count_memory_refs( + self, + gateway_id: str | None = None, + session_id: str | None = None, + status: BackendRefStatus | str | None = None, + ) -> int: + return len(self.list_memory_refs(gateway_id=gateway_id, session_id=session_id, status=status, limit=100000)) + + def get_commit_job(self, job_id: str) -> CommitJob | None: + return self.commit_jobs.get(job_id) + + def update_commit_job_status( + self, + job_id: str, + status: str, + error_message: str | None = None, + created_refs_count: int | None = None, + ) -> CommitJob | None: + job = self.commit_jobs.get(job_id) + if not job: + return None + from .schemas_v2 import OperationStatus + + job.status = OperationStatus(_enum_value(status)) + job.error_message = error_message + if created_refs_count is not None: + job.created_refs_count = created_refs_count + now = datetime.now(timezone.utc) + job.updated_at = now + if job.status.value == "running" and job.started_at is None: + job.started_at = now + if job.status.value in {"success", "failed", "partial_success"}: + job.finished_at = now + self.commit_jobs[job.job_id] = job + return job + class SQLiteRepository: def __init__(self, db_path: str | Path) -> None: @@ -171,8 +443,121 @@ class SQLiteRepository: created_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_audit_created ON audit_logs(created_at); + CREATE TABLE IF NOT EXISTS memory_refs ( + id TEXT PRIMARY KEY, + gateway_id TEXT NOT NULL, + workspace_id TEXT NOT NULL, + user_id TEXT NOT NULL, + agent_id TEXT, + session_id TEXT, + turn_id TEXT, + namespace TEXT, + backend_type TEXT NOT NULL, + ref_type TEXT NOT NULL, + native_id TEXT, + native_uri TEXT, + provenance_id TEXT, + idempotency_key TEXT, + content_hash TEXT, + source_type TEXT, + source_event_id TEXT, + status TEXT NOT NULL, + error_message TEXT, + metadata_json TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_memory_refs_gateway ON memory_refs(gateway_id); + CREATE INDEX IF NOT EXISTS idx_memory_refs_scope ON memory_refs(workspace_id, user_id, agent_id, session_id); + CREATE INDEX IF NOT EXISTS idx_memory_refs_backend ON memory_refs(backend_type, ref_type, status); + CREATE INDEX IF NOT EXISTS idx_memory_refs_namespace ON memory_refs(namespace); + CREATE TABLE IF NOT EXISTS outbox_events ( + id TEXT PRIMARY KEY, + event_type TEXT NOT NULL, + gateway_id TEXT NOT NULL, + workspace_id TEXT NOT NULL, + user_id TEXT NOT NULL, + agent_id TEXT, + session_id TEXT, + backend_type TEXT NOT NULL, + operation TEXT NOT NULL, + payload_ref TEXT, + status TEXT NOT NULL, + attempt_count INTEGER NOT NULL, + max_attempts INTEGER NOT NULL, + next_retry_at TEXT, + last_error TEXT, + locked_by TEXT, + locked_at TEXT, + lease_expires_at TEXT, + metadata_json TEXT NOT NULL, + payload TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_outbox_events_status ON outbox_events(status, next_retry_at); + CREATE INDEX IF NOT EXISTS idx_outbox_events_backend ON outbox_events(backend_type, operation); + CREATE INDEX IF NOT EXISTS idx_outbox_events_gateway ON outbox_events(gateway_id); + CREATE TABLE IF NOT EXISTS commit_jobs ( + job_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + user_id TEXT NOT NULL, + agent_id TEXT, + session_id TEXT NOT NULL, + namespace TEXT, + status TEXT NOT NULL, + requested_by TEXT, + created_refs_count INTEGER NOT NULL, + error_message TEXT, + payload TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + started_at TEXT, + finished_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_commit_jobs_session ON commit_jobs(session_id); + CREATE INDEX IF NOT EXISTS idx_commit_jobs_status ON commit_jobs(status); """ ) + self._ensure_memory_ref_columns(conn) + self._ensure_outbox_event_columns(conn) + conn.execute( + """ + DELETE FROM memory_refs + WHERE rowid NOT IN ( + SELECT MAX(rowid) + FROM memory_refs + GROUP BY gateway_id, backend_type, ref_type + ) + """ + ) + conn.execute( + """ + DROP INDEX IF EXISTS uq_memory_refs_gateway_backend_ref_type + """ + ) + + def _ensure_memory_ref_columns(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(memory_refs)").fetchall()} + additions = { + "idempotency_key": "TEXT", + "content_hash": "TEXT", + } + for column, column_type in additions.items(): + if column not in columns: + conn.execute(f"ALTER TABLE memory_refs ADD COLUMN {column} {column_type}") + + def _ensure_outbox_event_columns(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(outbox_events)").fetchall()} + additions = { + "locked_by": "TEXT", + "locked_at": "TEXT", + "lease_expires_at": "TEXT", + } + for column, column_type in additions.items(): + if column not in columns: + conn.execute(f"ALTER TABLE outbox_events ADD COLUMN {column} {column_type}") def create_user(self, user: UserRecord) -> UserRecord: now = datetime.now(timezone.utc) @@ -316,6 +701,397 @@ class SQLiteRepository: ).fetchall() return [_json_load_model(AuditLog, row["payload"]) for row in rows] + def save_memory_ref(self, ref: MemoryRef) -> MemoryRef: + existing = None + with self._connect() as conn: + row = conn.execute("SELECT payload FROM memory_refs WHERE id = ?", (ref.id,)).fetchone() + if row: + existing = _json_load_model(MemoryRef, row["payload"]) + now = datetime.now(timezone.utc) + if existing: + ref.created_at = existing.created_at + ref.updated_at = now + with self._connect() as conn: + conn.execute( + """ + INSERT OR REPLACE INTO memory_refs( + id, gateway_id, workspace_id, user_id, agent_id, session_id, turn_id, + namespace, backend_type, ref_type, native_id, native_uri, provenance_id, + idempotency_key, content_hash, source_type, source_event_id, status, error_message, metadata_json, + payload, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + ref.id, + ref.gateway_id, + ref.workspace_id, + ref.user_id, + ref.agent_id, + ref.session_id, + ref.turn_id, + ref.namespace, + ref.backend_type.value, + ref.ref_type.value, + ref.native_id, + ref.native_uri, + ref.provenance_id, + ref.idempotency_key, + ref.content_hash, + ref.source_type, + ref.source_event_id, + ref.status.value, + ref.error_message, + json.dumps(ref.metadata, ensure_ascii=False), + _json_dump_model(ref), + ref.created_at.isoformat(), + ref.updated_at.isoformat(), + ), + ) + return ref + + def get_memory_ref(self, ref_id: str) -> MemoryRef | None: + with self._connect() as conn: + row = conn.execute("SELECT payload FROM memory_refs WHERE id = ?", (ref_id,)).fetchone() + return _json_load_model(MemoryRef, row["payload"]) if row else None + + def list_memory_refs( + self, + gateway_id: str | None = None, + workspace_id: str | None = None, + user_id: str | None = None, + agent_id: str | None = None, + session_id: str | None = None, + namespace: str | None = None, + backend_type: BackendType | str | None = None, + ref_type: MemoryRefType | str | None = None, + status: BackendRefStatus | str | None = None, + limit: int = 100, + ) -> list[MemoryRef]: + clauses: list[str] = [] + params: list[str | int] = [] + filters = { + "gateway_id": gateway_id, + "workspace_id": workspace_id, + "user_id": user_id, + "agent_id": agent_id, + "session_id": session_id, + "namespace": namespace, + "backend_type": _enum_value(backend_type) if backend_type is not None else None, + "ref_type": _enum_value(ref_type) if ref_type is not None else None, + "status": _enum_value(status) if status is not None else None, + } + for key, value in filters.items(): + if value is not None: + clauses.append(f"{key} = ?") + params.append(value) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(limit) + with self._connect() as conn: + rows = conn.execute( + f"SELECT payload FROM memory_refs {where} ORDER BY created_at DESC LIMIT ?", + params, + ).fetchall() + return [_json_load_model(MemoryRef, row["payload"]) for row in rows] + + def save_outbox_event(self, event: OutboxEvent) -> OutboxEvent: + existing = None + with self._connect() as conn: + row = conn.execute("SELECT payload FROM outbox_events WHERE id = ?", (event.id,)).fetchone() + if row: + existing = _json_load_model(OutboxEvent, row["payload"]) + now = datetime.now(timezone.utc) + if existing: + event.created_at = existing.created_at + event.updated_at = now + with self._connect() as conn: + conn.execute( + """ + INSERT OR REPLACE INTO outbox_events( + id, event_type, gateway_id, workspace_id, user_id, agent_id, session_id, + backend_type, operation, payload_ref, status, attempt_count, max_attempts, + next_retry_at, last_error, locked_by, locked_at, lease_expires_at, + metadata_json, payload, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + event.id, + event.event_type, + event.gateway_id, + event.workspace_id, + event.user_id, + event.agent_id, + event.session_id, + event.backend_type.value, + event.operation.value, + event.payload_ref, + event.status.value, + event.attempt_count, + event.max_attempts, + event.next_retry_at.isoformat() if event.next_retry_at else None, + event.last_error, + event.locked_by, + event.locked_at.isoformat() if event.locked_at else None, + event.lease_expires_at.isoformat() if event.lease_expires_at else None, + json.dumps(event.metadata, ensure_ascii=False), + _json_dump_model(event), + event.created_at.isoformat(), + event.updated_at.isoformat(), + ), + ) + return event + + def list_outbox_events( + self, + status: OutboxEventStatus | str | None = None, + backend_type: BackendType | str | None = None, + operation: BackendOperation | str | None = None, + gateway_id: str | None = None, + payload_ref: str | None = None, + limit: int = 100, + ) -> list[OutboxEvent]: + clauses: list[str] = [] + params: list[str | int] = [] + filters = { + "status": _enum_value(status) if status is not None else None, + "backend_type": _enum_value(backend_type) if backend_type is not None else None, + "operation": _enum_value(operation) if operation is not None else None, + "gateway_id": gateway_id, + "payload_ref": payload_ref, + } + for key, value in filters.items(): + if value is not None: + clauses.append(f"{key} = ?") + params.append(value) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(limit) + with self._connect() as conn: + rows = conn.execute( + f"SELECT payload FROM outbox_events {where} ORDER BY created_at DESC LIMIT ?", + params, + ).fetchall() + return [_json_load_model(OutboxEvent, row["payload"]) for row in rows] + + def list_outbox_events_by_job(self, job_id: str, limit: int = 100) -> list[OutboxEvent]: + return self.list_outbox_events(payload_ref=f"commit_job:{job_id}", limit=limit) + + def claim_outbox_event(self, event_id: str) -> OutboxEvent | None: + with self._connect() as conn: + row = conn.execute("SELECT payload FROM outbox_events WHERE id = ?", (event_id,)).fetchone() + if not row: + return None + event = _json_load_model(OutboxEvent, row["payload"]) + now = datetime.now(timezone.utc) + if event.status != OutboxEventStatus.PENDING: + return None + if event.next_retry_at and event.next_retry_at > now: + return None + event.status = OutboxEventStatus.PROCESSING + event.locked_by = "inline" + event.locked_at = now + event.lease_expires_at = now + _safe_timedelta(300) + event.updated_at = now + with self._connect() as conn: + cursor = conn.execute( + """ + UPDATE outbox_events + SET status = ?, locked_by = ?, locked_at = ?, lease_expires_at = ?, + payload = ?, metadata_json = ?, updated_at = ? + WHERE id = ? + AND status = ? + AND (next_retry_at IS NULL OR next_retry_at <= ?) + """, + ( + event.status.value, + event.locked_by, + event.locked_at.isoformat() if event.locked_at else None, + event.lease_expires_at.isoformat() if event.lease_expires_at else None, + _json_dump_model(event), + json.dumps(event.metadata, ensure_ascii=False), + event.updated_at.isoformat(), + event.id, + OutboxEventStatus.PENDING.value, + now.isoformat(), + ), + ) + return event if cursor.rowcount else None + + def claim_pending_outbox_events( + self, + limit: int, + worker_id: str, + lease_seconds: int, + ) -> list[OutboxEvent]: + now = datetime.now(timezone.utc) + now_iso = now.isoformat() + with self._connect() as conn: + rows = conn.execute( + """ + SELECT payload FROM outbox_events + WHERE status = ? + AND (next_retry_at IS NULL OR next_retry_at <= ?) + ORDER BY created_at ASC + LIMIT ? + """, + (OutboxEventStatus.PENDING.value, now_iso, limit), + ).fetchall() + claimed: list[OutboxEvent] = [] + with self._connect() as conn: + for row in rows: + event = _json_load_model(OutboxEvent, row["payload"]) + if event.status != OutboxEventStatus.PENDING: + continue + event.status = OutboxEventStatus.PROCESSING + event.locked_by = worker_id + event.locked_at = now + event.lease_expires_at = now + _safe_timedelta(lease_seconds) + event.updated_at = now + cursor = conn.execute( + """ + UPDATE outbox_events + SET status = ?, locked_by = ?, locked_at = ?, lease_expires_at = ?, + payload = ?, metadata_json = ?, updated_at = ? + WHERE id = ? + AND status = ? + AND (next_retry_at IS NULL OR next_retry_at <= ?) + """, + ( + event.status.value, + event.locked_by, + event.locked_at.isoformat() if event.locked_at else None, + event.lease_expires_at.isoformat() if event.lease_expires_at else None, + _json_dump_model(event), + json.dumps(event.metadata, ensure_ascii=False), + event.updated_at.isoformat(), + event.id, + OutboxEventStatus.PENDING.value, + now_iso, + ), + ) + if cursor.rowcount: + claimed.append(event) + return claimed + + def release_expired_processing_events(self, now: datetime | None = None) -> list[OutboxEvent]: + now = now or datetime.now(timezone.utc) + with self._connect() as conn: + rows = conn.execute( + """ + SELECT payload FROM outbox_events + WHERE status = ? + AND lease_expires_at IS NOT NULL + AND lease_expires_at <= ? + """, + (OutboxEventStatus.PROCESSING.value, now.isoformat()), + ).fetchall() + released: list[OutboxEvent] = [] + for row in rows: + event = _json_load_model(OutboxEvent, row["payload"]) + event.status = OutboxEventStatus.PENDING + event.locked_by = None + event.locked_at = None + event.lease_expires_at = None + event.updated_at = now + released.append(self.save_outbox_event(event)) + return released + + def update_outbox_event_status( + self, + event_id: str, + status: OutboxEventStatus | str, + last_error: str | None = None, + ) -> OutboxEvent | None: + with self._connect() as conn: + row = conn.execute("SELECT payload FROM outbox_events WHERE id = ?", (event_id,)).fetchone() + if not row: + return None + event = _json_load_model(OutboxEvent, row["payload"]) + event.status = OutboxEventStatus(_enum_value(status)) + event.last_error = last_error + event.updated_at = datetime.now(timezone.utc) + if event.status != OutboxEventStatus.PROCESSING: + event.locked_by = None + event.locked_at = None + event.lease_expires_at = None + if event.status in {OutboxEventStatus.FAILED, OutboxEventStatus.DEAD_LETTER}: + event.attempt_count += 1 + return self.save_outbox_event(event) + + def save_commit_job(self, job: CommitJob) -> CommitJob: + existing = None + with self._connect() as conn: + row = conn.execute("SELECT payload FROM commit_jobs WHERE job_id = ?", (job.job_id,)).fetchone() + if row: + existing = _json_load_model(CommitJob, row["payload"]) + now = datetime.now(timezone.utc) + if existing: + job.created_at = existing.created_at + job.updated_at = now + with self._connect() as conn: + conn.execute( + """ + INSERT OR REPLACE INTO commit_jobs( + job_id, workspace_id, user_id, agent_id, session_id, namespace, + status, requested_by, created_refs_count, error_message, payload, + created_at, updated_at, started_at, finished_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + job.job_id, + job.workspace_id, + job.user_id, + job.agent_id, + job.session_id, + job.namespace, + job.status.value, + job.requested_by, + job.created_refs_count, + job.error_message, + _json_dump_model(job), + job.created_at.isoformat(), + job.updated_at.isoformat(), + job.started_at.isoformat() if job.started_at else None, + job.finished_at.isoformat() if job.finished_at else None, + ), + ) + return job + + def get_commit_job(self, job_id: str) -> CommitJob | None: + with self._connect() as conn: + row = conn.execute("SELECT payload FROM commit_jobs WHERE job_id = ?", (job_id,)).fetchone() + return _json_load_model(CommitJob, row["payload"]) if row else None + + def update_commit_job_status( + self, + job_id: str, + status: str, + error_message: str | None = None, + created_refs_count: int | None = None, + ) -> CommitJob | None: + from .schemas_v2 import OperationStatus + + job = self.get_commit_job(job_id) + if not job: + return None + job.status = OperationStatus(_enum_value(status)) + job.error_message = error_message + if created_refs_count is not None: + job.created_refs_count = created_refs_count + now = datetime.now(timezone.utc) + job.updated_at = now + if job.status.value == "running" and job.started_at is None: + job.started_at = now + if job.status.value in {"success", "failed", "partial_success"}: + job.finished_at = now + return self.save_commit_job(job) + + def count_memory_refs( + self, + gateway_id: str | None = None, + session_id: str | None = None, + status: BackendRefStatus | str | None = None, + ) -> int: + return len(self.list_memory_refs(gateway_id=gateway_id, session_id=session_id, status=status, limit=100000)) + def build_repository() -> MetadataRepository: config = get_config() @@ -325,4 +1101,3 @@ def build_repository() -> MetadataRepository: repository = build_repository() - diff --git a/memory_gateway/schemas_v2.py b/memory_gateway/schemas_v2.py new file mode 100644 index 0000000..58c8418 --- /dev/null +++ b/memory_gateway/schemas_v2.py @@ -0,0 +1,228 @@ +"""Schemas for the Memory Gateway v2 control-plane API.""" +from __future__ import annotations + +from datetime import datetime +from enum import Enum +from typing import Any, Literal, Optional +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from .schemas import utc_now + + +class OperationStatus(str, Enum): + ACCEPTED = "accepted" + RUNNING = "running" + SUCCESS = "success" + PARTIAL_SUCCESS = "partial_success" + FAILED = "failed" + PENDING = "pending" + SKIPPED = "skipped" + + +class BackendRefStatus(str, Enum): + PENDING = "pending" + SUCCESS = "success" + FAILED = "failed" + SKIPPED = "skipped" + + +class BackendType(str, Enum): + OPENVIKING = "openviking" + EVERMEMOS = "evermemos" + OBSIDIAN = "obsidian" + + +class MemoryRefType(str, Enum): + SESSION_ARCHIVE = "session_archive" + CONTEXT_RESOURCE = "context_resource" + MESSAGE_MEMORY = "message_memory" + EPISODIC_MEMORY = "episodic_memory" + PROFILE = "profile" + LONG_TERM_MEMORY = "long_term_memory" + DRAFT_REVIEW = "draft_review" + + +class TraceContext(BaseModel): + trace_id: Optional[str] = None + span_id: Optional[str] = None + parent_span_id: Optional[str] = None + request_id: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class IngestPolicy(BaseModel): + allow_openviking: bool = True + allow_evermemos: bool = True + allow_obsidian_review: bool = False + redact_sensitive: bool = True + require_human_review: bool = False + metadata: dict[str, Any] = Field(default_factory=dict) + + +class IngestRequest(BaseModel): + workspace_id: str + user_id: str + agent_id: str + session_id: str + turn_id: str + request_id: Optional[str] = None + idempotency_key: Optional[str] = None + namespace: str + source_type: str = "conversation" + source_event_id: Optional[str] = None + role: Literal["system", "user", "assistant", "tool", "agent"] = "user" + content: str + policy: IngestPolicy = Field(default_factory=IngestPolicy) + trace: TraceContext = Field(default_factory=TraceContext) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class MemoryRef(BaseModel): + id: str = Field(default_factory=lambda: f"ref_{uuid4().hex[:16]}") + gateway_id: str + workspace_id: str + user_id: str + agent_id: Optional[str] = None + session_id: Optional[str] = None + turn_id: Optional[str] = None + namespace: Optional[str] = None + backend_type: BackendType + ref_type: MemoryRefType + native_id: Optional[str] = None + native_uri: Optional[str] = None + provenance_id: Optional[str] = None + idempotency_key: Optional[str] = None + content_hash: Optional[str] = None + source_type: Optional[str] = None + source_event_id: Optional[str] = None + status: BackendRefStatus = BackendRefStatus.PENDING + error_message: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) + created_at: datetime = Field(default_factory=utc_now) + updated_at: datetime = Field(default_factory=utc_now) + + +class MemoryRefView(MemoryRef): + pass + + +class IngestResponse(BaseModel): + status: OperationStatus + gateway_id: str + provenance_id: str + request_id: Optional[str] = None + turn_id: str + refs: list[MemoryRefView] = Field(default_factory=list) + errors: list[str] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class CommitRequest(BaseModel): + workspace_id: str + user_id: str + agent_id: Optional[str] = None + namespace: Optional[str] = None + request_id: Optional[str] = None + idempotency_key: Optional[str] = None + policy: IngestPolicy = Field(default_factory=IngestPolicy) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class CommitResponse(BaseModel): + status: OperationStatus = OperationStatus.ACCEPTED + job_id: str + session_id: str + message: str = "commit accepted" + refs: list[MemoryRefView] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class OutboxSummary(BaseModel): + total_events: int = 0 + pending_events: int = 0 + processing_events: int = 0 + success_events: int = 0 + skipped_events: int = 0 + dead_letter_events: int = 0 + + +class CommitJobView(BaseModel): + job_id: str + workspace_id: str + user_id: str + agent_id: Optional[str] = None + session_id: str + namespace: Optional[str] = None + status: OperationStatus + created_refs_count: int = 0 + error_message: Optional[str] = None + created_at: datetime + updated_at: datetime + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + outbox_summary: OutboxSummary = Field(default_factory=OutboxSummary) + + +class OutboxProcessResponse(BaseModel): + status: OperationStatus + worker_id: str + processed_count: int = 0 + outbox_summary: OutboxSummary = Field(default_factory=OutboxSummary) + + +class RetrieveRequest(BaseModel): + workspace_id: str + user_id: str + agent_id: Optional[str] = None + session_id: Optional[str] = None + namespace: Optional[str] = None + query: str + limit: int = Field(default=10, ge=1, le=100) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class ContextItem(BaseModel): + text: Optional[str] = None + source_backend: BackendType + ref_id: Optional[str] = None + score: float = 0.0 + memory_type: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class ContextConflict(BaseModel): + ref_ids: list[str] = Field(default_factory=list) + reason: str + metadata: dict[str, Any] = Field(default_factory=dict) + + +class RetrieveResponse(BaseModel): + status: OperationStatus + items: list[ContextItem] = Field(default_factory=list) + refs: list[MemoryRefView] = Field(default_factory=list) + conflicts: list[ContextConflict] = Field(default_factory=list) + trace_id: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FeedbackRequest(BaseModel): + workspace_id: str + user_id: str + agent_id: Optional[str] = None + session_id: Optional[str] = None + namespace: Optional[str] = None + memory_ref_id: Optional[str] = None + feedback_type: Literal["useful", "not_useful", "incorrect", "duplicate", "outdated", "review_approved", "review_rejected"] + comment: Optional[str] = None + source_type: str = "manual" + source_event_id: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FeedbackResponse(BaseModel): + status: OperationStatus + feedback_id: str + memory_ref_id: Optional[str] = None + metadata: dict[str, Any] = Field(default_factory=dict) diff --git a/memory_gateway/server.py b/memory_gateway/server.py index abe793a..1793a4b 100644 --- a/memory_gateway/server.py +++ b/memory_gateway/server.py @@ -572,8 +572,10 @@ app.include_router(mcp_router, prefix="/mcp", tags=["mcp"]) # Generic Memory Gateway v1 routes are imported lazily here to avoid changing # the existing legacy /api and /mcp startup path. from .api_v1 import router as api_v1_router # noqa: E402 +from .api_v2 import router as api_v2_router # noqa: E402 app.include_router(api_v1_router) +app.include_router(api_v2_router) @app.post("/api/search", dependencies=[Depends(verify_api_key)]) diff --git a/memory_gateway/services_v2.py b/memory_gateway/services_v2.py new file mode 100644 index 0000000..05559c8 --- /dev/null +++ b/memory_gateway/services_v2.py @@ -0,0 +1,970 @@ +"""Service orchestration for the Memory Gateway v2 control plane.""" +from __future__ import annotations + +import hashlib +from datetime import datetime, timedelta, timezone +from typing import Any, Awaitable, Callable +from uuid import uuid4 + +from fastapi import HTTPException, status + +from .backend_contracts import ( + BackendOperation, + BackendCommitResult, + BackendProducedRef, + BackendResultStatus, + BackendWriteResult, + CommitJob, + OutboxEvent, + OutboxEventStatus, +) +from .evermemos_client import EverMemOSClient +from .openviking_client import get_openviking_client +from .repositories import MetadataRepository, repository +from .schemas import AuditLog +from .schemas_v2 import ( + BackendRefStatus, + BackendType, + CommitJobView, + CommitRequest, + CommitResponse, + ContextItem, + FeedbackRequest, + FeedbackResponse, + IngestRequest, + IngestResponse, + MemoryRef, + MemoryRefView, + MemoryRefType, + OperationStatus, + OutboxProcessResponse, + OutboxSummary, + RetrieveRequest, + RetrieveResponse, +) + + +OpenVikingClientFactory = Callable[[], Awaitable[Any]] + + +class MemoryGatewayV2Service: + def __init__( + self, + repo: MetadataRepository = repository, + openviking_client_factory: OpenVikingClientFactory = get_openviking_client, + evermemos_client: Any | None = None, + ) -> None: + self.repo = repo + self.openviking_client_factory = openviking_client_factory + self.evermemos_client = evermemos_client + + async def ingest_conversation_turn(self, request: IngestRequest) -> IngestResponse: + normalized = self._normalize_ingest_request(request) + gateway_id = self._build_gateway_id(normalized) + provenance_id = self._build_provenance_id(normalized, gateway_id) + content_hash = self._content_hash(normalized.content) + + self._check_namespace_access(normalized) + payload = self._apply_safety_policy(normalized) + + refs: list[MemoryRef] = [] + if normalized.policy.allow_openviking: + refs.append( + await self._write_openviking_turn( + normalized, + payload, + gateway_id=gateway_id, + provenance_id=provenance_id, + content_hash=content_hash, + ) + ) + else: + refs.append( + self._save_ref( + normalized, + gateway_id, + provenance_id, + BackendType.OPENVIKING, + MemoryRefType.SESSION_ARCHIVE, + BackendRefStatus.SKIPPED, + content_hash=content_hash, + metadata=self._control_metadata(normalized, content_hash, {"reason": "policy_disabled"}), + ) + ) + + if normalized.policy.allow_evermemos: + refs.append( + await self._write_evermemos_message( + normalized, + payload, + gateway_id=gateway_id, + provenance_id=provenance_id, + content_hash=content_hash, + ) + ) + else: + refs.append( + self._save_ref( + normalized, + gateway_id, + provenance_id, + BackendType.EVERMEMOS, + MemoryRefType.MESSAGE_MEMORY, + BackendRefStatus.SKIPPED, + content_hash=content_hash, + metadata=self._control_metadata(normalized, content_hash, {"reason": "policy_disabled"}), + ) + ) + + status_value = self._aggregate_ref_status(refs) + errors = [ref.error_message for ref in refs if ref.error_message] + self.repo.add_audit( + AuditLog( + actor_user_id=normalized.user_id, + actor_agent_id=normalized.agent_id, + action="v2_ingest_conversation_turn", + target_type="conversation_turn", + target_id=normalized.turn_id, + namespace=normalized.namespace, + metadata={ + "gateway_id": gateway_id, + "provenance_id": provenance_id, + "idempotency_basis": self._idempotency_basis(normalized), + "content_hash": content_hash, + "status": status_value.value, + "source_type": normalized.source_type, + "source_event_id": normalized.source_event_id, + }, + ) + ) + return IngestResponse( + status=status_value, + gateway_id=gateway_id, + provenance_id=provenance_id, + request_id=normalized.request_id, + turn_id=normalized.turn_id, + refs=self._view_refs(refs), + errors=errors, + metadata={"backend_count": len(refs)}, + ) + + async def commit_session(self, session_id: str, request: CommitRequest) -> CommitResponse: + # TODO(v2): add a worker that consumes these outbox events and writes + # resulting backend refs. This method intentionally only records + # control-plane intent. + job_id = f"job_{uuid4().hex[:16]}" + gateway_id = self._commit_gateway_id(session_id, request) + job = CommitJob( + job_id=job_id, + workspace_id=request.workspace_id, + user_id=request.user_id, + agent_id=request.agent_id, + session_id=session_id, + namespace=request.namespace, + requested_by=request.user_id, + ) + self.repo.save_commit_job(job) + self._create_commit_outbox_events(gateway_id, job, request) + self.repo.add_audit( + AuditLog( + actor_user_id=request.user_id, + actor_agent_id=request.agent_id, + action="v2_commit_session_accepted", + target_type="session", + target_id=session_id, + namespace=request.namespace, + metadata={ + "job_id": job_id, + "gateway_id": gateway_id, + "workspace_id": request.workspace_id, + "outbox_events": 2, + }, + ) + ) + return CommitResponse( + job_id=job_id, + session_id=session_id, + metadata={"gateway_id": gateway_id, "outbox_events": 2}, + ) + + async def retrieve_context(self, request: RetrieveRequest) -> RetrieveResponse: + # TODO(v2): expand namespace ACL, fan out concurrently to OpenViking and + # EverMemOS, then apply lightweight merge/rerank before returning. + refs = self.repo.list_memory_refs( + workspace_id=request.workspace_id, + user_id=request.user_id, + agent_id=request.agent_id, + session_id=request.session_id, + namespace=request.namespace, + limit=request.limit, + ) + items = [ + ContextItem( + text=None, + source_backend=ref.backend_type, + ref_id=ref.id, + score=0.0, + memory_type=ref.ref_type.value, + metadata={ + "status": ref.status.value, + "native_id": ref.native_id, + "native_uri": ref.native_uri, + }, + ) + for ref in refs + ] + trace_id = request.metadata.get("trace_id") if request.metadata else None + return RetrieveResponse( + status=OperationStatus.SUCCESS, + items=items, + refs=self._view_refs(refs), + conflicts=[], + trace_id=trace_id, + metadata={"skeleton": True}, + ) + + async def record_memory_feedback(self, request: FeedbackRequest) -> FeedbackResponse: + # TODO(v2): persist review/feedback state in a dedicated table and route + # accepted corrections back to the owning backend adapter. + feedback_id = f"fb_{uuid4().hex[:16]}" + self.repo.add_audit( + AuditLog( + actor_user_id=request.user_id, + actor_agent_id=request.agent_id, + action=f"v2_feedback:{request.feedback_type}", + target_type="memory_ref", + target_id=request.memory_ref_id, + namespace=request.namespace, + metadata={ + "feedback_id": feedback_id, + "workspace_id": request.workspace_id, + "comment": request.comment, + "source_type": request.source_type, + "source_event_id": request.source_event_id, + }, + ) + ) + return FeedbackResponse( + status=OperationStatus.ACCEPTED, + feedback_id=feedback_id, + memory_ref_id=request.memory_ref_id, + ) + + async def process_pending_outbox_events( + self, + limit: int = 100, + worker_id: str | None = None, + lease_seconds: int = 300, + ) -> list[OutboxEvent]: + worker_id = worker_id or f"worker_{uuid4().hex[:12]}" + self.repo.release_expired_processing_events() + events = self.repo.claim_pending_outbox_events( + limit=limit, + worker_id=worker_id, + lease_seconds=lease_seconds, + ) + processed: list[OutboxEvent] = [] + for event in events: + try: + result = await self._execute_outbox_event(event) + except Exception as exc: # noqa: BLE001 + result = BackendCommitResult( + backend_type=event.backend_type, + operation=event.operation, + status=BackendResultStatus.FAILED, + retryable=True, + error_code="adapter_exception", + error_message=str(exc), + ) + processed.append(self._apply_outbox_result(event, result)) + return processed + + async def process_pending_outbox_events_summary( + self, + limit: int = 100, + worker_id: str | None = None, + lease_seconds: int = 300, + ) -> OutboxProcessResponse: + worker_id = worker_id or f"worker_{uuid4().hex[:12]}" + processed = await self.process_pending_outbox_events( + limit=limit, + worker_id=worker_id, + lease_seconds=lease_seconds, + ) + return OutboxProcessResponse( + status=OperationStatus.SUCCESS, + worker_id=worker_id, + processed_count=len(processed), + outbox_summary=self._outbox_summary(), + ) + + async def process_commit_job(self, job_id: str) -> CommitJob: + job = self.repo.get_commit_job(job_id) + if not job: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Commit job not found") + + self.repo.update_commit_job_status(job_id, OperationStatus.RUNNING.value) + self.repo.release_expired_processing_events() + events = self.repo.list_outbox_events_by_job(job_id) + for event in events: + if event.status == OutboxEventStatus.PENDING: + await self.process_outbox_event(event.id) + + events = self.repo.list_outbox_events_by_job(job_id) + final_status = self._aggregate_commit_job_status(events) + created_refs_count = self.repo.count_memory_refs(session_id=job.session_id, status=BackendRefStatus.SUCCESS) + error_message = self._commit_job_error_message(events) + updated = self.repo.update_commit_job_status( + job_id, + final_status.value, + error_message=error_message, + created_refs_count=created_refs_count, + ) + if not updated: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Commit job not found") + return updated + + def get_commit_job_view(self, job_id: str) -> CommitJobView: + job = self.repo.get_commit_job(job_id) + if not job: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Commit job not found") + return CommitJobView( + job_id=job.job_id, + workspace_id=job.workspace_id, + user_id=job.user_id, + agent_id=job.agent_id, + session_id=job.session_id, + namespace=job.namespace, + status=job.status, + created_refs_count=job.created_refs_count, + error_message=job.error_message, + created_at=job.created_at, + updated_at=job.updated_at, + started_at=job.started_at, + finished_at=job.finished_at, + outbox_summary=self._outbox_summary(self.repo.list_outbox_events_by_job(job_id)), + ) + + async def process_outbox_event(self, event_id: str) -> OutboxEvent | None: + event = self.repo.claim_outbox_event(event_id) + if not event: + return None + try: + result = await self._execute_outbox_event(event) + except Exception as exc: # noqa: BLE001 + result = BackendCommitResult( + backend_type=event.backend_type, + operation=event.operation, + status=BackendResultStatus.FAILED, + retryable=True, + error_code="adapter_exception", + error_message=str(exc), + ) + return self._apply_outbox_result(event, result) + + def list_memory_refs( + self, + workspace_id: str | None = None, + user_id: str | None = None, + agent_id: str | None = None, + session_id: str | None = None, + namespace: str | None = None, + backend_type: BackendType | str | None = None, + ref_type: MemoryRefType | str | None = None, + status: BackendRefStatus | str | None = None, + limit: int = 100, + ) -> list[MemoryRef]: + return self.repo.list_memory_refs( + workspace_id=workspace_id, + user_id=user_id, + agent_id=agent_id, + session_id=session_id, + namespace=namespace, + backend_type=backend_type, + ref_type=ref_type, + status=status, + limit=limit, + ) + + async def _execute_outbox_event(self, event: OutboxEvent) -> BackendCommitResult | BackendWriteResult: + payload = self._outbox_payload(event) + if event.operation != BackendOperation.COMMIT_SESSION: + return BackendCommitResult( + backend_type=event.backend_type, + operation=event.operation, + status=BackendResultStatus.SKIPPED, + metadata={"reason": "unsupported_operation"}, + ) + if event.backend_type == BackendType.OPENVIKING: + client = await self.openviking_client_factory() + if not hasattr(client, "commit_session_v2"): + return BackendCommitResult( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.COMMIT_SESSION, + status=BackendResultStatus.SKIPPED, + metadata={"reason": "adapter_method_missing"}, + ) + result = await client.commit_session_v2(payload) + return result + if event.backend_type == BackendType.EVERMEMOS: + client = self.evermemos_client or EverMemOSClient() + if not hasattr(client, "extract_profile_long_term_v2"): + return BackendCommitResult( + backend_type=BackendType.EVERMEMOS, + operation=BackendOperation.COMMIT_SESSION, + status=BackendResultStatus.SKIPPED, + metadata={"reason": "adapter_method_missing"}, + ) + result = client.extract_profile_long_term_v2(payload) + if hasattr(result, "__await__"): + result = await result + return result + return BackendCommitResult( + backend_type=event.backend_type, + operation=event.operation, + status=BackendResultStatus.SKIPPED, + metadata={"reason": "unsupported_backend"}, + ) + + def _outbox_payload(self, event: OutboxEvent) -> dict[str, Any]: + return { + "event_id": event.id, + "gateway_id": event.gateway_id, + "workspace_id": event.workspace_id, + "user_id": event.user_id, + "agent_id": event.agent_id, + "session_id": event.session_id, + "backend_type": event.backend_type.value, + "operation": event.operation.value, + "payload_ref": event.payload_ref, + "metadata": self._safe_control_metadata(event.metadata), + } + + def _apply_outbox_result(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> OutboxEvent: + result_data = self._backend_result_to_dict(result) + safe_result = self._backend_control_metadata(result_data) + event.metadata = self._safe_control_metadata({**event.metadata, "backend_result": safe_result}) + event.last_error = result.error_message + event.updated_at = datetime.now(timezone.utc) + + if result.status == BackendResultStatus.SUCCESS: + self._write_commit_memory_refs(event, result) + event.status = OutboxEventStatus.SUCCESS + self._clear_outbox_lock(event) + return self.repo.save_outbox_event(event) + + if result.status == BackendResultStatus.SKIPPED: + event.status = OutboxEventStatus.SKIPPED + self._clear_outbox_lock(event) + return self.repo.save_outbox_event(event) + + if result.status == BackendResultStatus.FAILED: + return self._handle_failed_outbox_event(event, result) + + event.status = OutboxEventStatus.PENDING + self._clear_outbox_lock(event) + return self.repo.save_outbox_event(event) + + def _handle_failed_outbox_event(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> OutboxEvent: + event.attempt_count += 1 + event.last_error = result.error_message + if result.retryable and event.attempt_count < event.max_attempts: + event.status = OutboxEventStatus.PENDING + event.next_retry_at = datetime.now(timezone.utc) + timedelta(seconds=min(60, 2**event.attempt_count)) + else: + event.status = OutboxEventStatus.DEAD_LETTER + event.next_retry_at = None + self._clear_outbox_lock(event) + return self.repo.save_outbox_event(event) + + def _clear_outbox_lock(self, event: OutboxEvent) -> None: + event.locked_by = None + event.locked_at = None + event.lease_expires_at = None + + def _write_commit_memory_refs(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> list[MemoryRef]: + produced_refs = result.refs if isinstance(result, BackendCommitResult) and result.refs else [] + if produced_refs: + refs: list[MemoryRef] = [] + for index, produced_ref in enumerate(produced_refs): + saved = self._write_commit_memory_ref(event, result, produced_ref, index=index) + if saved: + refs.append(saved) + return refs + + fallback = BackendProducedRef( + ref_type=self._commit_ref_type(event, result), + native_id=result.native_id, + native_uri=result.native_uri, + metadata={}, + ) + saved = self._write_commit_memory_ref(event, result, fallback, index=0) + return [saved] if saved else [] + + def _write_commit_memory_ref( + self, + event: OutboxEvent, + result: BackendCommitResult | BackendWriteResult, + produced_ref: BackendProducedRef, + index: int, + ) -> MemoryRef | None: + stable_key = self._produced_ref_stable_key(produced_ref, index) + if not produced_ref.native_id and not produced_ref.native_uri and not stable_key: + return None + ref_type = produced_ref.ref_type + ref_id = self._memory_ref_id(event.gateway_id, event.backend_type, ref_type, produced_ref.native_id, produced_ref.native_uri, stable_key) + existing = self.repo.get_memory_ref(ref_id) + safe_produced_metadata = self._safe_control_metadata(produced_ref.metadata) + ref = MemoryRef( + id=ref_id, + gateway_id=event.gateway_id, + workspace_id=event.workspace_id, + user_id=event.user_id, + agent_id=event.agent_id, + session_id=event.session_id, + namespace=event.metadata.get("namespace"), + backend_type=event.backend_type, + ref_type=ref_type, + native_id=produced_ref.native_id, + native_uri=produced_ref.native_uri, + provenance_id="prov_" + + hashlib.sha256(f"{event.id}|{ref_type.value}|{produced_ref.native_id}|{produced_ref.native_uri}".encode("utf-8")).hexdigest()[:24], + source_type="commit_session", + source_event_id=event.id, + status=BackendRefStatus.SUCCESS, + error_message=None, + metadata={ + "schema_version": "memory-gateway.commit-ref.v2", + "job_id": self._job_id_from_payload_ref(event.payload_ref), + "outbox_event_id": event.id, + "operation": event.operation.value, + "produced_ref_index": index, + "stable_key": stable_key, + "produced_ref": safe_produced_metadata, + "backend_result": self._backend_control_metadata(self._backend_result_to_dict(result)), + }, + ) + if existing: + ref.created_at = existing.created_at + return self.repo.save_memory_ref(ref) + + def _commit_ref_type(self, event: OutboxEvent, result: BackendCommitResult | BackendWriteResult) -> MemoryRefType: + requested = result.metadata.get("ref_type") if isinstance(result.metadata, dict) else None + if requested: + try: + return MemoryRefType(requested) + except ValueError: + pass + if event.backend_type == BackendType.OPENVIKING: + return MemoryRefType.SESSION_ARCHIVE + if event.backend_type == BackendType.EVERMEMOS: + return MemoryRefType.LONG_TERM_MEMORY + return MemoryRefType.DRAFT_REVIEW + + def _job_id_from_payload_ref(self, payload_ref: str | None) -> str | None: + if payload_ref and payload_ref.startswith("commit_job:"): + return payload_ref.split(":", 1)[1] + return None + + def _aggregate_commit_job_status(self, events: list[OutboxEvent]) -> OperationStatus: + if not events: + return OperationStatus.FAILED + statuses = {event.status for event in events} + if statuses.issubset({OutboxEventStatus.SUCCESS, OutboxEventStatus.SKIPPED}): + return OperationStatus.SUCCESS + if statuses == {OutboxEventStatus.DEAD_LETTER} or statuses == {OutboxEventStatus.FAILED}: + return OperationStatus.FAILED + if OutboxEventStatus.PENDING in statuses or OutboxEventStatus.PROCESSING in statuses: + if OutboxEventStatus.SUCCESS in statuses: + return OperationStatus.PARTIAL_SUCCESS + return OperationStatus.FAILED + if OutboxEventStatus.SUCCESS in statuses: + return OperationStatus.PARTIAL_SUCCESS + return OperationStatus.FAILED + + def _commit_job_error_message(self, events: list[OutboxEvent]) -> str | None: + errors = [event.last_error for event in events if event.last_error] + return "; ".join(errors) if errors else None + + def _outbox_summary(self, events: list[OutboxEvent] | None = None) -> OutboxSummary: + events = events if events is not None else self.repo.list_outbox_events(limit=100000) + counts = {status: 0 for status in OutboxEventStatus} + for event in events: + counts[event.status] = counts.get(event.status, 0) + 1 + return OutboxSummary( + total_events=len(events), + pending_events=counts.get(OutboxEventStatus.PENDING, 0), + processing_events=counts.get(OutboxEventStatus.PROCESSING, 0), + success_events=counts.get(OutboxEventStatus.SUCCESS, 0), + skipped_events=counts.get(OutboxEventStatus.SKIPPED, 0), + dead_letter_events=counts.get(OutboxEventStatus.DEAD_LETTER, 0), + ) + + def _safe_control_metadata(self, metadata: dict[str, Any] | None) -> dict[str, Any]: + if not metadata: + return {} + blocked = {"content", "raw_request", "messages", "conversation", "transcript"} + safe: dict[str, Any] = {} + for key, value in metadata.items(): + if key in blocked: + continue + if isinstance(value, dict): + nested = self._safe_control_metadata(value) + if nested: + safe[key] = nested + elif isinstance(value, (str, int, float, bool)) or value is None: + safe[key] = value + elif isinstance(value, list): + safe[key] = [item for item in value if isinstance(item, (str, int, float, bool))] + return safe + + def _normalize_ingest_request(self, request: IngestRequest) -> IngestRequest: + data = request.model_copy(deep=True) + data.namespace = data.namespace.strip("/") + data.content = data.content.strip() + if not data.namespace: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="namespace is required") + if not data.content: + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="content is required") + return data + + def _build_gateway_id(self, request: IngestRequest) -> str: + seed = self._idempotency_basis(request) + return "gw_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] + + def _build_provenance_id(self, request: IngestRequest, gateway_id: str) -> str: + seed = f"{gateway_id}|{self._idempotency_basis(request)}" + return "prov_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] + + def _idempotency_basis(self, request: IngestRequest) -> str: + if request.idempotency_key: + return f"idempotency_key:{request.workspace_id}:{request.idempotency_key}" + if request.source_event_id: + return f"source_event_id:{request.workspace_id}:{request.source_type}:{request.source_event_id}" + return f"turn:{request.workspace_id}:{request.session_id}:{request.turn_id}" + + def _content_hash(self, content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + def _check_namespace_access(self, request: IngestRequest) -> None: + # TODO(v2): enforce workspace/user/agent namespace ACL tree. + if not request.workspace_id or not request.user_id or not request.agent_id: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="workspace, user, and agent are required") + + def _apply_safety_policy(self, request: IngestRequest) -> dict[str, Any]: + # TODO(v2): apply configurable redaction/safety rules before fan-out. + return { + "workspace_id": request.workspace_id, + "user_id": request.user_id, + "agent_id": request.agent_id, + "session_id": request.session_id, + "turn_id": request.turn_id, + "namespace": request.namespace, + "source_type": request.source_type, + "source_event_id": request.source_event_id, + "role": request.role, + "content": request.content, + "metadata": request.metadata, + "trace": request.trace.model_dump(mode="json"), + } + + async def _write_openviking_turn( + self, + request: IngestRequest, + payload: dict[str, Any], + gateway_id: str, + provenance_id: str, + content_hash: str, + ) -> MemoryRef: + try: + client = await self.openviking_client_factory() + if not hasattr(client, "ingest_conversation_turn"): + return self._save_ref( + request, + gateway_id, + provenance_id, + BackendType.OPENVIKING, + MemoryRefType.SESSION_ARCHIVE, + BackendRefStatus.SKIPPED, + content_hash=content_hash, + metadata=self._control_metadata(request, content_hash, {"reason": "adapter_method_missing"}), + ) + result = await client.ingest_conversation_turn(payload) + return self._ref_from_backend_result( + request, + gateway_id, + provenance_id, + BackendType.OPENVIKING, + MemoryRefType.SESSION_ARCHIVE, + result, + content_hash, + ) + except Exception as exc: # noqa: BLE001 + return self._save_ref( + request, + gateway_id, + provenance_id, + BackendType.OPENVIKING, + MemoryRefType.SESSION_ARCHIVE, + BackendRefStatus.FAILED, + content_hash=content_hash, + error_message=str(exc), + metadata=self._control_metadata(request, content_hash), + ) + + async def _write_evermemos_message( + self, + request: IngestRequest, + payload: dict[str, Any], + gateway_id: str, + provenance_id: str, + content_hash: str, + ) -> MemoryRef: + try: + client = self.evermemos_client or EverMemOSClient() + if not hasattr(client, "ingest_message"): + return self._save_ref( + request, + gateway_id, + provenance_id, + BackendType.EVERMEMOS, + MemoryRefType.MESSAGE_MEMORY, + BackendRefStatus.SKIPPED, + content_hash=content_hash, + metadata=self._control_metadata(request, content_hash, {"reason": "adapter_method_missing"}), + ) + result = client.ingest_message(payload) + if hasattr(result, "__await__"): + result = await result + return self._ref_from_backend_result( + request, + gateway_id, + provenance_id, + BackendType.EVERMEMOS, + MemoryRefType.MESSAGE_MEMORY, + result, + content_hash, + ) + except Exception as exc: # noqa: BLE001 + return self._save_ref( + request, + gateway_id, + provenance_id, + BackendType.EVERMEMOS, + MemoryRefType.MESSAGE_MEMORY, + BackendRefStatus.FAILED, + content_hash=content_hash, + error_message=str(exc), + metadata=self._control_metadata(request, content_hash), + ) + + def _ref_from_backend_result( + self, + request: IngestRequest, + gateway_id: str, + provenance_id: str, + backend_type: BackendType, + ref_type: MemoryRefType, + result: Any, + content_hash: str, + ) -> MemoryRef: + data = self._backend_result_to_dict(result) + raw_status = str(data.get("status") or "success") + ref_status = BackendRefStatus.SUCCESS + if raw_status in {BackendRefStatus.PENDING.value, BackendRefStatus.FAILED.value, BackendRefStatus.SKIPPED.value}: + ref_status = BackendRefStatus(raw_status) + native_id = data.get("native_id") or data.get("id") or data.get("memory_id") or data.get("session_id") + native_uri = data.get("native_uri") or data.get("uri") or data.get("url") + return self._save_ref( + request, + gateway_id, + provenance_id, + backend_type, + ref_type, + ref_status, + native_id=native_id, + native_uri=native_uri, + error_message=data.get("error") or data.get("error_message"), + content_hash=content_hash, + metadata=self._control_metadata( + request, + content_hash, + {"backend_response": self._backend_control_metadata(data)}, + ), + ) + + def _save_ref( + self, + request: IngestRequest, + gateway_id: str, + provenance_id: str, + backend_type: BackendType, + ref_type: MemoryRefType, + ref_status: BackendRefStatus, + native_id: str | None = None, + native_uri: str | None = None, + error_message: str | None = None, + content_hash: str | None = None, + metadata: dict[str, Any] | None = None, + ) -> MemoryRef: + ref_id = self._memory_ref_id(gateway_id, backend_type, ref_type) + existing = self.repo.get_memory_ref(ref_id) + ref = MemoryRef( + id=ref_id, + gateway_id=gateway_id, + workspace_id=request.workspace_id, + user_id=request.user_id, + agent_id=request.agent_id, + session_id=request.session_id, + turn_id=request.turn_id, + namespace=request.namespace, + backend_type=backend_type, + ref_type=ref_type, + native_id=native_id, + native_uri=native_uri, + provenance_id=provenance_id, + idempotency_key=request.idempotency_key, + content_hash=content_hash, + source_type=request.source_type, + source_event_id=request.source_event_id, + status=ref_status, + error_message=error_message, + metadata=metadata or {}, + ) + if existing: + ref.created_at = existing.created_at + return self.repo.save_memory_ref(ref) + + def _memory_ref_id( + self, + gateway_id: str, + backend_type: BackendType, + ref_type: MemoryRefType, + native_id: str | None = None, + native_uri: str | None = None, + stable_key: str | None = None, + ) -> str: + native_key = f"|{native_id or ''}|{native_uri or ''}|{stable_key or ''}" if native_id or native_uri or stable_key else "" + seed = f"{gateway_id}|{backend_type.value}|{ref_type.value}{native_key}" + return "ref_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] + + def _produced_ref_stable_key(self, produced_ref: BackendProducedRef, index: int) -> str | None: + for key in ("stable_key", "backend_ref_key", "idempotency_key"): + value = produced_ref.metadata.get(key) + if isinstance(value, str) and value: + return value + if not produced_ref.native_id and not produced_ref.native_uri: + return f"produced_ref_index:{index}" + return None + + def _aggregate_ref_status(self, refs: list[MemoryRef]) -> OperationStatus: + if not refs: + return OperationStatus.SKIPPED + statuses = {ref.status for ref in refs} + if statuses == {BackendRefStatus.SUCCESS}: + return OperationStatus.SUCCESS + if BackendRefStatus.SUCCESS in statuses and (BackendRefStatus.FAILED in statuses or BackendRefStatus.SKIPPED in statuses): + return OperationStatus.PARTIAL_SUCCESS + if statuses == {BackendRefStatus.FAILED}: + return OperationStatus.FAILED + if BackendRefStatus.FAILED in statuses: + return OperationStatus.PARTIAL_SUCCESS + if statuses == {BackendRefStatus.SKIPPED}: + return OperationStatus.SKIPPED + return OperationStatus.PENDING + + def _view_refs(self, refs: list[MemoryRef]) -> list[MemoryRefView]: + return [MemoryRefView.model_validate(ref.model_dump(mode="json")) for ref in refs] + + def _backend_control_metadata(self, data: dict[str, Any]) -> dict[str, Any]: + allowed_keys = { + "status", + "reason", + "native_id", + "native_uri", + "id", + "uri", + "url", + "memory_id", + "session_id", + "retryable", + "error_code", + "error", + "error_message", + "latency_ms", + } + metadata = {key: value for key, value in data.items() if key in allowed_keys} + nested_metadata = data.get("metadata") + if isinstance(nested_metadata, dict): + for key in ("reason", "backend_request_id", "latency_ms", "schema_version"): + if key in nested_metadata: + metadata[key] = nested_metadata[key] + return metadata + + def _control_metadata( + self, + request: IngestRequest, + content_hash: str, + extra: dict[str, Any] | None = None, + ) -> dict[str, Any]: + metadata: dict[str, Any] = { + "schema_version": "memory-gateway.control-ref.v2", + "idempotency_basis": self._idempotency_basis(request), + "content_hash": content_hash, + } + source_channel = request.metadata.get("source_channel") or request.metadata.get("channel") + if source_channel: + metadata["source_channel"] = source_channel + if request.trace.trace_id: + metadata["trace_id"] = request.trace.trace_id + if request.trace.request_id: + metadata["trace_request_id"] = request.trace.request_id + if extra: + metadata.update(extra) + return metadata + + def _backend_result_to_dict(self, result: Any) -> dict[str, Any]: + if isinstance(result, BackendWriteResult): + data = result.model_dump(mode="json") + if result.status == BackendResultStatus.FAILED and result.retryable: + data["retryable"] = True + return data + if hasattr(result, "model_dump"): + return result.model_dump(mode="json") + return result if isinstance(result, dict) else {"raw": str(result)} + + def _commit_gateway_id(self, session_id: str, request: CommitRequest) -> str: + basis = request.idempotency_key or request.request_id or session_id + seed = f"commit:{request.workspace_id}:{session_id}:{basis}" + return "gwc_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] + + def _create_commit_outbox_events(self, gateway_id: str, job: CommitJob, request: CommitRequest) -> None: + metadata = { + "schema_version": "memory-gateway.outbox.v2", + "job_id": job.job_id, + "namespace": request.namespace, + "idempotency_key": request.idempotency_key, + "request_id": request.request_id, + } + for backend_type in (BackendType.OPENVIKING, BackendType.EVERMEMOS): + event = OutboxEvent( + id=self._outbox_event_id(gateway_id, backend_type, BackendOperation.COMMIT_SESSION), + event_type="commit_session", + gateway_id=gateway_id, + workspace_id=request.workspace_id, + user_id=request.user_id, + agent_id=request.agent_id, + session_id=job.session_id, + backend_type=backend_type, + operation=BackendOperation.COMMIT_SESSION, + payload_ref=f"commit_job:{job.job_id}", + metadata=metadata, + ) + self.repo.save_outbox_event(event) + + def _outbox_event_id(self, gateway_id: str, backend_type: BackendType, operation: BackendOperation) -> str: + seed = f"{gateway_id}|{backend_type.value}|{operation.value}" + return "outbox_" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] + + +v2_service = MemoryGatewayV2Service() diff --git a/memory_gateway/types.py b/memory_gateway/types.py index 33dbeb8..5c6826b 100644 --- a/memory_gateway/types.py +++ b/memory_gateway/types.py @@ -12,18 +12,25 @@ class ServerConfig(BaseModel): class OpenVikingConfig(BaseModel): """OpenViking 后端配置""" + enabled: bool = False + mode: Literal["offline", "skeleton", "real"] = "offline" url: str = "http://localhost:1933" api_key: str = "" timeout: int = 30 + verify_ssl: bool = True + ingest_path: str = "/api/v1/sessions/{session_id}/messages" class EverMemOSConfig(BaseModel): """External EverMemOS consolidation service configuration.""" - enabled: bool = True + enabled: bool = False + mode: Literal["offline", "skeleton", "real"] = "offline" url: str = "http://127.0.0.1:1995" api_key: str = "" timeout: int = 30 + verify_ssl: bool = True health_path: str = "/health" + ingest_path: str = "/api/v1/memories" consolidate_path: str = "/v1/sessions/consolidate" fallback_to_local: bool = True diff --git a/memory_gateway/worker_v2.py b/memory_gateway/worker_v2.py new file mode 100644 index 0000000..f2b782f --- /dev/null +++ b/memory_gateway/worker_v2.py @@ -0,0 +1,49 @@ +"""Lightweight v2 outbox worker entrypoint. + +Usage: + python -m memory_gateway.worker_v2 --limit 100 --worker-id local-worker --lease-seconds 300 +""" +from __future__ import annotations + +import argparse +import asyncio +import json +from typing import Sequence +from uuid import uuid4 + +from .services_v2 import v2_service + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Process Memory Gateway v2 outbox events once.") + parser.add_argument("--limit", type=int, default=100, help="Maximum pending events to claim and process.") + parser.add_argument("--worker-id", default=None, help="Stable worker id recorded in outbox lease fields.") + parser.add_argument("--lease-seconds", type=int, default=300, help="Lease duration for claimed events.") + return parser + + +async def run_once(limit: int, worker_id: str | None, lease_seconds: int) -> dict[str, object]: + worker_id = worker_id or f"worker_{uuid4().hex[:12]}" + response = await v2_service.process_pending_outbox_events_summary( + limit=limit, + worker_id=worker_id, + lease_seconds=lease_seconds, + ) + return response.model_dump(mode="json") + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + payload = asyncio.run( + run_once( + limit=args.limit, + worker_id=args.worker_id, + lease_seconds=args.lease_seconds, + ) + ) + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/test_real_ingest.py b/tests/integration/test_real_ingest.py new file mode 100644 index 0000000..7dc1fc8 --- /dev/null +++ b/tests/integration/test_real_ingest.py @@ -0,0 +1,102 @@ +import asyncio +import os +from uuid import uuid4 + +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +import memory_gateway.api_v2 as api_v2 +from memory_gateway.evermemos_client import EverMemOSClient +from memory_gateway.openviking_client import OpenVikingClient +from memory_gateway.repositories import InMemoryRepository +from memory_gateway.schemas_v2 import BackendRefStatus, BackendType, IngestRequest, IngestResponse, OperationStatus +from memory_gateway.server_auth import verify_api_key_compat +from memory_gateway.services_v2 import MemoryGatewayV2Service + + +pytestmark = pytest.mark.skipif( + os.environ.get("RUN_REAL_BACKEND_TESTS") != "1", + reason="real backend ingest test is opt-in; set RUN_REAL_BACKEND_TESTS=1", +) + + +def _env(name: str) -> str: + value = os.environ.get(name) + if not value: + pytest.skip(f"{name} is required for real backend ingest test") + return value + + +def test_real_openviking_and_evermemos_ingest_writes_memory_refs(): + openviking_base_url = _env("OPENVIKING_BASE_URL") + evermemos_base_url = _env("EVERMEMOS_BASE_URL") + openviking_api_key = os.environ.get("OPENVIKING_API_KEY", "") + evermemos_api_key = os.environ.get("EVERMEMOS_API_KEY", "") + openviking_ingest_path = os.environ.get("OPENVIKING_INGEST_PATH") + evermemos_ingest_path = os.environ.get("EVERMEMOS_INGEST_PATH") + + async def openviking_factory(): + return OpenVikingClient( + mode="real", + base_url=openviking_base_url, + api_key=openviking_api_key, + ingest_path=openviking_ingest_path, + ) + + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=openviking_factory, + evermemos_client=EverMemOSClient( + mode="real", + base_url=evermemos_base_url, + api_key=evermemos_api_key, + ingest_path=evermemos_ingest_path, + ), + ) + run_id = uuid4().hex[:12] + + response = asyncio.run(post_ingest(service, run_id)) + + refs = repo.list_memory_refs(session_id=f"real_ingest_sess_{run_id}", limit=10) + assert {ref.backend_type for ref in refs} == {BackendType.OPENVIKING, BackendType.EVERMEMOS} + assert all(ref.content_hash for ref in refs) + openviking_ref = next(ref for ref in refs if ref.backend_type == BackendType.OPENVIKING) + evermemos_ref = next(ref for ref in refs if ref.backend_type == BackendType.EVERMEMOS) + + assert openviking_ref.status == BackendRefStatus.SUCCESS + if evermemos_ref.status == BackendRefStatus.SUCCESS: + assert response.status == OperationStatus.SUCCESS + assert evermemos_ref.native_id + assert evermemos_ref.native_uri + else: + assert evermemos_ref.status == BackendRefStatus.FAILED + assert response.status == OperationStatus.PARTIAL_SUCCESS + assert evermemos_ref.error_message + + +async def post_ingest(service: MemoryGatewayV2Service, run_id: str): + api_v2.v2_service = service + app = FastAPI() + app.dependency_overrides[verify_api_key_compat] = lambda: None + app.include_router(api_v2.router) + request = IngestRequest( + workspace_id=os.environ.get("REAL_BACKEND_WORKSPACE_ID", "ws_real_ingest"), + user_id=os.environ.get("REAL_BACKEND_USER_ID", "user_real_ingest"), + agent_id=os.environ.get("REAL_BACKEND_AGENT_ID", "agent_real_ingest"), + session_id=f"real_ingest_sess_{run_id}", + turn_id=f"real_ingest_turn_{run_id}", + request_id=f"real_ingest_req_{run_id}", + idempotency_key=f"real_ingest_idem_{run_id}", + namespace=os.environ.get("REAL_BACKEND_NAMESPACE", "workspace/ws_real_ingest/user/user_real_ingest"), + source_type="integration_test", + source_event_id=f"real_ingest_evt_{run_id}", + role="user", + content=f"Memory Gateway real ingest smoke test {run_id}", + metadata={"source_channel": "integration_test"}, + ) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + response = await client.post("/v2/conversations/ingest", json=request.model_dump(mode="json")) + response.raise_for_status() + return IngestResponse.model_validate(response.json()) diff --git a/tests/test_server.py b/tests/test_server.py index 8a5f5c7..0b2fdbb 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,8 +1,10 @@ +import asyncio import sys import types +import pytest +from fastapi import HTTPException from fastapi.responses import StreamingResponse -from fastapi.testclient import TestClient def install_test_stubs() -> None: @@ -59,8 +61,8 @@ def install_test_stubs() -> None: install_test_stubs() -from memory_gateway.server import app -from memory_gateway.types import Config, ObsidianConfig, SearchResult, ServerConfig +import memory_gateway.server as server +from memory_gateway.types import CommitSummaryRequest, Config, ObsidianConfig, SearchRequest, SearchResult, ServerConfig class FakeOVClient: @@ -117,9 +119,13 @@ async def fake_summarize_with_llm(content, **kwargs): } -def build_headers(api_key: str | None): - return {"x-api-key": api_key} if api_key is not None else {} +class FakeUploadFile: + def __init__(self, filename: str, content: bytes) -> None: + self.filename = filename + self._content = content + async def read(self) -> bytes: + return self._content def test_health_requires_api_key(monkeypatch): monkeypatch.setattr( @@ -131,14 +137,15 @@ def test_health_requires_api_key(monkeypatch): fake_get_openviking_client, ) monkeypatch.setattr("memory_gateway.server.summarize_with_llm", fake_summarize_with_llm) + monkeypatch.setattr("memory_gateway.server.v1_service.evermemos_health", lambda: {"status": "disabled"}) - with TestClient(app) as client: - response = client.get("/health") - assert response.status_code == 401 + with pytest.raises(HTTPException) as exc_info: + server.verify_api_key() + assert exc_info.value.status_code == 401 - response = client.get("/health", headers=build_headers("secret")) - assert response.status_code == 200 - assert response.json()["openviking"]["status"] == "ok" + server.verify_api_key("secret") + payload = asyncio.run(server.health_check()) + assert payload["openviking"]["status"] == "ok" def test_mcp_rpc_lists_tools_with_api_key(monkeypatch): @@ -151,18 +158,11 @@ def test_mcp_rpc_lists_tools_with_api_key(monkeypatch): fake_get_openviking_client, ) - with TestClient(app) as client: - response = client.post( - "/mcp/rpc", - json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, - headers=build_headers("secret"), - ) - assert response.status_code == 200 - payload = response.json() - assert payload["jsonrpc"] == "2.0" - assert len(payload["result"]["tools"]) >= 7 - assert any(tool["name"] == "commit_summary" for tool in payload["result"]["tools"]) - assert any(tool["name"] == "memory_search" for tool in payload["result"]["tools"]) + server.verify_api_key("secret") + tools = asyncio.run(server.list_tools()) + assert len(tools) >= 7 + assert any(tool.name == "commit_summary" for tool in tools) + assert any(tool.name == "memory_search" for tool in tools) def test_search_passes_through_gateway(monkeypatch): @@ -175,12 +175,9 @@ def test_search_passes_through_gateway(monkeypatch): fake_get_openviking_client, ) - with TestClient(app) as client: - response = client.post("/api/search", json={"query": "phishing"}) - assert response.status_code == 200 - payload = response.json() - assert payload["total"] == 1 - assert payload["results"][0]["abstract"] == "phishing" + payload = asyncio.run(server.api_search(SearchRequest(query="phishing"))) + assert payload["total"] == 1 + assert payload["results"][0]["abstract"] == "phishing" def test_summary_endpoint_builds_generic_artifact(monkeypatch): @@ -194,28 +191,26 @@ def test_summary_endpoint_builds_generic_artifact(monkeypatch): ) monkeypatch.setattr("memory_gateway.server.summarize_with_llm", fake_summarize_with_llm) - with TestClient(app) as client: - response = client.post( - "/api/summary", - json={ - "title": "Demo investigation summary", - "content": "结论：这是一次高价值沉淀。\n- 证据：命中历史 case。\n- 建议：后续复用该处置路径。", - "namespace": "demo", - "memory_type": "knowledge", - "tags": ["demo", "summary"], - "persist_as": "none", - }, + payload = asyncio.run( + server.api_commit_summary( + CommitSummaryRequest( + title="Demo investigation summary", + content="结论：这是一次高价值沉淀。\n- 证据：命中历史 case。\n- 建议：后续复用该处置路径。", + namespace="demo", + memory_type="knowledge", + tags=["demo", "summary"], + persist_as="none", + ) ) - assert response.status_code == 200 - payload = response.json() - assert payload["status"] == "ok" - assert payload["artifact"]["title"] == "Demo investigation summary" - assert payload["artifact"]["namespace"] == "demo" - assert payload["artifact"]["memory_type"] == "knowledge" - assert payload["artifact"]["summary"].startswith("LLM summary:") - assert payload["artifact"]["llm"]["provider"] == "fake" - assert payload["memory_result"] is None - assert payload["resource_result"] is None + ) + assert payload["status"] == "ok" + assert payload["artifact"]["title"] == "Demo investigation summary" + assert payload["artifact"]["namespace"] == "demo" + assert payload["artifact"]["memory_type"] == "knowledge" + assert payload["artifact"]["summary"].startswith("LLM summary:") + assert payload["artifact"]["llm"]["provider"] == "fake" + assert payload["memory_result"] is None + assert payload["resource_result"] is None def test_knowledge_upload_converts_saves_and_commits(monkeypatch, tmp_path): @@ -230,21 +225,27 @@ def test_knowledge_upload_converts_saves_and_commits(monkeypatch, tmp_path): monkeypatch.setattr("memory_gateway.server.summarize_with_llm", fake_summarize_with_llm) monkeypatch.setattr("memory_gateway.server.convert_file_to_markdown", lambda path: "# Uploaded Doc\n\nImportant uploaded knowledge.") - with TestClient(app) as client: - response = client.post( - "/api/knowledge/upload", - data={ - "title": "Uploaded Knowledge", - "namespace": "demo", - "knowledge_type": "playbook", - "tags": "demo,upload", - "persist_as": "resource", - }, - files={"file": ("sample.txt", b"hello", "text/plain")}, - ) + async def fake_to_thread(func, *args, **kwargs): + return func(*args, **kwargs) + + monkeypatch.setattr("memory_gateway.server.asyncio.to_thread", fake_to_thread) + + upload = FakeUploadFile(filename="sample.txt", content=b"hello") + payload = asyncio.run( + server.api_upload_knowledge( + file=upload, + title="Uploaded Knowledge", + namespace="demo", + knowledge_type="playbook", + tags="demo,upload", + source=None, + obsidian_dir=None, + resource_uri=None, + persist_as="resource", + max_summary_chars=1000, + ) + ) - assert response.status_code == 200 - payload = response.json() assert payload["status"] == "ok" assert payload["artifact"]["schema_version"] == "memory-gateway.knowledge_upload.v1" assert payload["artifact"]["knowledge_type"] == "playbook" diff --git a/tests/test_v2_api.py b/tests/test_v2_api.py new file mode 100644 index 0000000..6763109 --- /dev/null +++ b/tests/test_v2_api.py @@ -0,0 +1,1925 @@ +import asyncio +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path +from fastapi import FastAPI +import httpx +from httpx import ASGITransport, AsyncClient + +from memory_gateway.config import load_config +from memory_gateway.backend_adapter_mapping import ( + ADAPTER_MAPPING_SPECS, + DISALLOWED_PAYLOAD_FIELDS, + validate_control_plane_persisted_payload, + get_adapter_mapping_spec, + validate_control_plane_payload, +) +from memory_gateway.backend_normalization import ( + map_backend_error_to_retryable, + normalize_evermemos_commit_response, + normalize_evermemos_ingest_response, + normalize_evermemos_retrieve_response, + normalize_openviking_commit_response, + normalize_openviking_ingest_response, + normalize_openviking_retrieve_response, +) +from memory_gateway.backend_contracts import ( + BackendCommitResult, + BackendOperation, + BackendProducedRef, + BackendResultStatus, + BackendRetrieveResult, + BackendWriteResult, + OutboxEventStatus, +) +from memory_gateway.backend_ref_mapping import map_backend_ref_type +from memory_gateway.evermemos_client import EverMemOSClient +from memory_gateway.obsidian_review_client import ObsidianReviewClient +from memory_gateway.openviking_client import OpenVikingClient +from memory_gateway.repositories import InMemoryRepository, SQLiteRepository +from memory_gateway.schemas_v2 import ( + BackendRefStatus, + BackendType, + CommitRequest, + IngestRequest, + MemoryRefType, + OperationStatus, + OutboxProcessResponse, + RetrieveRequest, +) +from memory_gateway.server_auth import verify_api_key_compat +from memory_gateway.services_v2 import MemoryGatewayV2Service + + +FIXTURE_DIR = Path(__file__).parent / "fixtures" / "backend_responses" +DOCS_DIR = Path(__file__).parent.parent / "docs" + + +def load_backend_fixture(name: str): + return json.loads((FIXTURE_DIR / name).read_text()) + + +def build_ingest_payload(**overrides): + payload = { + "workspace_id": "ws_1", + "user_id": "user_a", + "agent_id": "agent_cli", + "session_id": "sess_1", + "turn_id": "turn_1", + "request_id": "req_1", + "namespace": "workspace/ws_1/user/user_a", + "source_type": "cli", + "source_event_id": "evt_1", + "role": "user", + "content": "Need to remember this conversation turn.", + "metadata": {"channel": "test"}, + } + payload.update(overrides) + return payload + + +class FakeOpenVikingClient: + async def ingest_conversation_turn(self, payload): + return { + "status": "success", + "native_id": f"ov_{payload['turn_id']}", + "native_uri": f"viking://sessions/{payload['session_id']}/{payload['turn_id']}", + } + + +async def fake_openviking_factory(): + return FakeOpenVikingClient() + + +class FakeEverMemOSClient: + def ingest_message(self, payload): + return { + "status": "success", + "native_id": f"em_{payload['turn_id']}", + "native_uri": f"evermemos://memories/{payload['turn_id']}", + } + + +class FailingEverMemOSClient: + def ingest_message(self, payload): + raise RuntimeError("evermemos unavailable") + + +class FakeCommitOpenVikingClient: + def __init__(self, result: BackendCommitResult) -> None: + self.result = result + + async def commit_session_v2(self, payload): + return self.result + + +def fake_commit_openviking_factory(result: BackendCommitResult): + async def factory(): + return FakeCommitOpenVikingClient(result) + + return factory + + +class FakeCommitEverMemOSClient: + def __init__(self, result: BackendCommitResult) -> None: + self.result = result + + def extract_profile_long_term_v2(self, payload): + return self.result + + +def commit_result( + backend_type: BackendType, + status: BackendResultStatus, + native_id: str | None = None, + native_uri: str | None = None, + retryable: bool = False, + error_message: str | None = None, +): + return BackendCommitResult( + backend_type=backend_type, + operation=BackendOperation.COMMIT_SESSION, + status=status, + native_id=native_id, + native_uri=native_uri, + retryable=retryable, + error_message=error_message, + ) + + +def test_v2_adapters_return_backend_write_result_contract(): + ov_result = asyncio.run( + OpenVikingClient().ingest_conversation_turn( + { + "workspace_id": "ws_1", + "session_id": "sess_1", + "turn_id": "turn_1", + } + ) + ) + em_result = EverMemOSClient().ingest_message( + { + "workspace_id": "ws_1", + "session_id": "sess_1", + "turn_id": "turn_1", + } + ) + + assert isinstance(ov_result, BackendWriteResult) + assert isinstance(em_result, BackendWriteResult) + assert ov_result.backend_type == BackendType.OPENVIKING + assert em_result.backend_type == BackendType.EVERMEMOS + assert ov_result.operation == BackendOperation.INGEST_TURN + assert em_result.operation == BackendOperation.INGEST_TURN + assert ov_result.status == BackendResultStatus.SKIPPED + assert em_result.status == BackendResultStatus.SKIPPED + + +def test_backend_env_overrides_enable_real_modes(monkeypatch, tmp_path): + monkeypatch.setenv("OPENVIKING_MODE", "real") + monkeypatch.setenv("OPENVIKING_BASE_URL", "http://openviking.env.test") + monkeypatch.setenv("OPENVIKING_API_KEY", "ov-env-token") + monkeypatch.setenv("OPENVIKING_TIMEOUT_SECONDS", "17") + monkeypatch.setenv("EVERMEMOS_MODE", "real") + monkeypatch.setenv("EVERMEMOS_BASE_URL", "http://evermemos.env.test") + monkeypatch.setenv("EVERMEMOS_API_KEY", "em-env-token") + monkeypatch.setenv("EVERMEMOS_INGEST_PATH", "/api/v1/memories") + + config = load_config(str(tmp_path / "missing.yaml")) + + assert config.openviking.mode == "real" + assert config.openviking.url == "http://openviking.env.test" + assert config.openviking.api_key == "ov-env-token" + assert config.openviking.timeout == 17 + assert config.evermemos.mode == "real" + assert config.evermemos.url == "http://evermemos.env.test" + assert config.evermemos.api_key == "em-env-token" + assert config.evermemos.ingest_path == "/api/v1/memories" + + +def test_openviking_default_ingest_does_not_touch_network(): + def handler(request): + raise AssertionError("offline OpenViking ingest should not perform HTTP") + + client = OpenVikingClient( + base_url="http://openviking.test", + transport=httpx.MockTransport(handler), + ) + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "sess_offline", "turn_id": "turn_1"})) + + assert result.status == BackendResultStatus.SKIPPED + assert result.native_uri == "viking://sessions/sess_offline" + + +def test_openviking_adapter_config_doc_exists_and_covers_modes_and_security(): + doc = (DOCS_DIR / "openviking_adapter_config.md").read_text() + + assert "offline" in doc + assert "real" in doc + assert "base_url" in doc + assert "api_key" in doc + assert "verify_ssl" in doc + assert "ingest_path" in doc + assert "content" in doc + assert "messages" in doc + assert "transcript" in doc + + +def test_openviking_mode_offline_does_not_touch_network_even_with_base_url(): + def handler(request): + raise AssertionError("offline mode should not perform HTTP") + + client = OpenVikingClient( + mode="offline", + base_url="http://openviking.test", + enabled=False, + transport=httpx.MockTransport(handler), + ) + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "sess_offline_mode", "turn_id": "turn_1"})) + + assert result.status == BackendResultStatus.SKIPPED + + +def test_openviking_mode_skeleton_does_not_touch_network_even_with_base_url(): + def handler(request): + raise AssertionError("skeleton mode should not perform HTTP") + + client = OpenVikingClient( + mode="skeleton", + base_url="http://openviking.test", + enabled=False, + transport=httpx.MockTransport(handler), + ) + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "sess_skeleton_mode", "turn_id": "turn_1"})) + + assert result.status == BackendResultStatus.SKIPPED + + +def test_openviking_mode_real_with_base_url_uses_mock_http(): + calls = {"count": 0} + + def handler(request): + calls["count"] += 1 + return httpx.Response(200, json=load_backend_fixture("openviking_ingest_real_success.json")) + + client = OpenVikingClient( + mode="real", + enabled=False, + base_url="http://openviking.test", + transport=httpx.MockTransport(handler), + ) + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "ov_real_sess_fixture_1", "turn_id": "turn_real", "content": "SECRET"})) + + assert calls["count"] == 1 + assert result.status == BackendResultStatus.SUCCESS + + +def test_openviking_enabled_true_without_mode_real_does_not_touch_network(): + seen = {"calls": 0} + + def handler(request): + seen["calls"] += 1 + raise AssertionError("enabled=True must not perform HTTP without mode=real") + + client = OpenVikingClient( + mode="offline", + enabled=True, + base_url="http://openviking.test", + transport=httpx.MockTransport(handler), + ) + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "ov_real_sess_fixture_1", "turn_id": "turn_x", "content": "SECRET"})) + + assert seen["calls"] == 0 + assert result.status == BackendResultStatus.SKIPPED + + +def test_openviking_real_ingest_mode_real_without_base_url_returns_config_error(): + client = OpenVikingClient(mode="real", base_url="") + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "sess_missing_url", "content": "SECRET"})) + + assert result.status == BackendResultStatus.FAILED + assert result.retryable is False + assert result.error_code == "config_error" + assert "SECRET" not in json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + + +def test_openviking_real_ingest_success_uses_mock_http_and_normalization(): + seen_payload = {} + seen_headers = {} + fixture = load_backend_fixture("openviking_ingest_real_success.json") + + def handler(request): + seen_payload.update(json.loads(request.content.decode())) + seen_headers.update(dict(request.headers)) + return httpx.Response( + 200, + json=fixture, + ) + + client = OpenVikingClient( + mode="real", + base_url="http://openviking.test", + api_key="token", + transport=httpx.MockTransport(handler), + ) + + result = asyncio.run( + client.ingest_conversation_turn( + { + "workspace_id": "ws_1", + "session_id": "ov_real_sess_fixture_1", + "turn_id": "turn_real", + "content": "SECRET_REAL_CONTENT", + } + ) + ) + expected = normalize_openviking_ingest_response(fixture) + + assert seen_payload["content"] == "SECRET_REAL_CONTENT" + assert seen_headers["x-api-key"] == "token" + assert result == expected + assert result.status == BackendResultStatus.SUCCESS + serialized = json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + assert "SECRET_REAL_CONTENT" not in serialized + assert "content" not in serialized + assert "token" not in serialized + + +def test_openviking_real_ingest_timeout_is_retryable_and_safe(): + def handler(request): + raise httpx.ReadTimeout("timeout while sending SECRET_TIMEOUT_CONTENT") + + client = OpenVikingClient( + mode="real", + base_url="http://openviking.test", + transport=httpx.MockTransport(handler), + ) + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "sess_timeout", "content": "SECRET_TIMEOUT_CONTENT"})) + + assert result.status == BackendResultStatus.FAILED + assert result.retryable is True + assert result.error_code == "timeout" + assert "SECRET_TIMEOUT_CONTENT" not in json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + + +def test_openviking_real_ingest_http_retryable_and_nonretryable_statuses(): + def client_for_fixture(name, status_code): + return OpenVikingClient( + mode="real", + base_url="http://openviking.test", + api_key="super-secret-token", + transport=httpx.MockTransport(lambda request: httpx.Response(status_code, json=load_backend_fixture(name))), + ) + + result_429 = asyncio.run(client_for_fixture("openviking_ingest_real_error_500.json", 429).ingest_conversation_turn({"session_id": "sess_http"})) + assert result_429.status == BackendResultStatus.FAILED + assert result_429.retryable is True + assert result_429.error_code == "http_429" + + result_500 = asyncio.run(client_for_fixture("openviking_ingest_real_error_500.json", 500).ingest_conversation_turn({"session_id": "sess_http"})) + assert result_500.status == BackendResultStatus.FAILED + assert result_500.retryable is True + assert result_500.error_code == "http_500" + assert "super-secret-token" not in json.dumps(result_500.model_dump(mode="json"), ensure_ascii=False) + + for name, status_code in ( + ("openviking_ingest_real_error_401.json", 401), + ("openviking_ingest_real_error_401.json", 403), + ("openviking_ingest_real_error_422.json", 422), + ): + result = asyncio.run(client_for_fixture(name, status_code).ingest_conversation_turn({"session_id": "sess_http"})) + assert result.status == BackendResultStatus.FAILED + assert result.retryable is False + assert result.error_code == f"http_{status_code}" + + +def test_openviking_real_ingest_invalid_json_returns_failed_retryable(): + client = OpenVikingClient( + mode="real", + base_url="http://openviking.test", + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"not-json")), + ) + + result = asyncio.run(client.ingest_conversation_turn({"session_id": "sess_invalid_json", "content": "SECRET_JSON"})) + + assert result.status == BackendResultStatus.FAILED + assert result.retryable is True + assert result.error_code == "invalid_json" + assert "SECRET_JSON" not in json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + + +def test_evermemos_default_ingest_does_not_touch_network_even_if_enabled(): + def handler(request): + raise AssertionError("EverMemOS ingest should not perform HTTP unless mode=real") + + client = EverMemOSClient( + enabled=True, + mode="offline", + base_url="http://evermemos.test", + transport=httpx.MockTransport(handler), + ) + + result = client.ingest_message({"session_id": "sess_offline", "turn_id": "turn_1", "content": "SECRET"}) + + assert result.status == BackendResultStatus.SKIPPED + + +def test_evermemos_real_ingest_mode_real_without_base_url_returns_config_error(): + client = EverMemOSClient(mode="real", base_url="") + + result = client.ingest_message({"session_id": "sess_missing_url", "content": "SECRET"}) + + assert result.status == BackendResultStatus.FAILED + assert result.retryable is False + assert result.error_code == "config_error" + assert "SECRET" not in json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + + +def test_evermemos_real_ingest_success_uses_mock_http_and_normalization(): + seen_payload = {} + seen_headers = {} + fixture = load_backend_fixture("evermemos_ingest_success.json") + + def handler(request): + seen_payload.update(json.loads(request.content.decode())) + seen_headers.update(dict(request.headers)) + return httpx.Response(200, json=fixture) + + client = EverMemOSClient( + mode="real", + base_url="http://evermemos.test", + api_key="em-token", + transport=httpx.MockTransport(handler), + ) + + result = client.ingest_message( + { + "workspace_id": "ws_1", + "user_id": "user_a", + "session_id": "sess_1", + "turn_id": "turn_1", + "role": "user", + "content": "SECRET_EM_CONTENT", + "source_type": "cli", + "source_event_id": "evt_1", + "metadata": {"channel": "test"}, + } + ) + expected = normalize_evermemos_ingest_response(fixture) + + assert seen_payload["content"] == "SECRET_EM_CONTENT" + assert seen_headers["x-api-key"] == "em-token" + assert seen_headers["authorization"] == "Bearer em-token" + assert result == expected + serialized = json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + assert "SECRET_EM_CONTENT" not in serialized + assert "content" not in serialized + assert "em-token" not in serialized + + +def test_evermemos_real_ingest_errors_are_backend_write_results_and_safe(): + def client_for_response(status_code, body=None, content=None): + return EverMemOSClient( + mode="real", + base_url="http://evermemos.test", + api_key="em-super-secret-token", + transport=httpx.MockTransport(lambda request: httpx.Response(status_code, json=body, content=content)), + ) + + result_500 = client_for_response(500, {"error_code": "server_error"}).ingest_message({"content": "SECRET"}) + assert result_500.status == BackendResultStatus.FAILED + assert result_500.retryable is True + assert result_500.error_code == "http_500" + + for status_code in (401, 403, 422): + result = client_for_response(status_code, {"error_code": "auth_or_validation"}).ingest_message({"content": "SECRET"}) + assert result.status == BackendResultStatus.FAILED + assert result.retryable is False + assert result.error_code == f"http_{status_code}" + + invalid = client_for_response(200, content=b"not-json").ingest_message({"content": "SECRET"}) + assert invalid.status == BackendResultStatus.FAILED + assert invalid.retryable is True + assert invalid.error_code == "invalid_json" + + serialized = json.dumps( + [result_500.model_dump(mode="json"), invalid.model_dump(mode="json")], + ensure_ascii=False, + ) + assert "SECRET" not in serialized + assert "em-super-secret-token" not in serialized + + +def test_evermemos_real_ingest_timeout_is_retryable_and_safe(): + def handler(request): + raise httpx.ReadTimeout("timeout while sending SECRET_TIMEOUT_CONTENT") + + client = EverMemOSClient( + mode="real", + base_url="http://evermemos.test", + transport=httpx.MockTransport(handler), + ) + + result = client.ingest_message({"session_id": "sess_timeout", "content": "SECRET_TIMEOUT_CONTENT"}) + + assert result.status == BackendResultStatus.FAILED + assert result.retryable is True + assert result.error_code == "timeout" + assert "SECRET_TIMEOUT_CONTENT" not in json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + + +def test_backend_adapter_mapping_spec_is_contract_first_and_control_plane_only(): + expected = { + (BackendType.OPENVIKING, BackendOperation.INGEST_TURN), + (BackendType.OPENVIKING, BackendOperation.COMMIT_SESSION), + (BackendType.OPENVIKING, BackendOperation.RETRIEVE_CONTEXT), + (BackendType.EVERMEMOS, BackendOperation.INGEST_TURN), + (BackendType.EVERMEMOS, BackendOperation.COMMIT_SESSION), + (BackendType.EVERMEMOS, BackendOperation.RETRIEVE_CONTEXT), + (BackendType.OBSIDIAN, BackendOperation.CREATE_REVIEW_DRAFT), + } + + assert {(spec.backend_type, spec.operation) for spec in ADAPTER_MAPPING_SPECS} == expected + for spec in ADAPTER_MAPPING_SPECS: + assert not DISALLOWED_PAYLOAD_FIELDS.intersection(spec.allowed_payload_fields) + + openviking_commit = get_adapter_mapping_spec(BackendType.OPENVIKING, BackendOperation.COMMIT_SESSION) + evermemos_ingest = get_adapter_mapping_spec(BackendType.EVERMEMOS, BackendOperation.INGEST_TURN) + + assert openviking_commit.adapter_method == "commit_session_v2" + assert openviking_commit.result_model is BackendCommitResult + assert evermemos_ingest.adapter_method == "ingest_message" + assert evermemos_ingest.result_model is BackendWriteResult + + +def test_control_plane_persisted_payload_validator_rejects_content_and_raw_request(): + validate_control_plane_payload({"gateway_id": "gw_1", "session_id": "sess_1", "metadata": {"content_hash": "abc"}}) + validate_control_plane_persisted_payload({"gateway_id": "gw_1", "metadata": {"source_channel": "test"}}) + + for blocked_key in ("content", "raw_request", "messages"): + try: + validate_control_plane_persisted_payload({"gateway_id": "gw_1", blocked_key: "should-not-pass"}) + except ValueError as exc: + assert blocked_key in str(exc) + else: + raise AssertionError(f"{blocked_key} should be rejected") + + +def test_runtime_adapter_request_may_be_transient_but_outbox_payload_is_control_plane_only(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service(repo=repo) + runtime_payload = service._apply_safety_policy(IngestRequest(**build_ingest_payload(content="TRANSIENT_ONLY_CONTENT"))) + + assert runtime_payload["content"] == "TRANSIENT_ONLY_CONTENT" + + response = asyncio.run( + service.commit_session("sess_boundary", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = repo.list_outbox_events_by_job(response.job_id)[0] + outbox_payload = service._outbox_payload(event) + + assert "content" not in outbox_payload + assert "raw_request" not in outbox_payload + validate_control_plane_persisted_payload(outbox_payload) + + +def test_commit_and_retrieve_adapter_skeletons_return_unified_contracts(): + payload = {"workspace_id": "ws_1", "session_id": "sess_1", "gateway_id": "gw_1"} + + ov_commit = asyncio.run(OpenVikingClient().commit_session_v2(payload)) + ov_retrieve = asyncio.run(OpenVikingClient().retrieve_context_v2(payload)) + em_commit = EverMemOSClient().extract_profile_long_term_v2(payload) + em_retrieve = EverMemOSClient().retrieve_context_v2(payload) + + assert isinstance(ov_commit, BackendCommitResult) + assert isinstance(em_commit, BackendCommitResult) + assert isinstance(ov_retrieve, BackendRetrieveResult) + assert isinstance(em_retrieve, BackendRetrieveResult) + assert ov_commit.status == BackendResultStatus.SUCCESS + assert em_commit.status == BackendResultStatus.SUCCESS + assert ov_retrieve.status == BackendResultStatus.SUCCESS + assert em_retrieve.status == BackendResultStatus.SUCCESS + assert ov_commit.refs[0].ref_type == MemoryRefType.SESSION_ARCHIVE + assert {ref.ref_type for ref in em_commit.refs} == {MemoryRefType.PROFILE, MemoryRefType.LONG_TERM_MEMORY} + assert len(ov_retrieve.items) == 1 + assert len(em_retrieve.items) == 2 + + +def test_client_skeletons_use_normalization_contracts_and_safe_metadata(): + payload = { + "workspace_id": "ws_1", + "user_id": "user_a", + "session_id": "sess_contract", + "turn_id": "turn_contract", + "content": "TRANSIENT_CONTENT_ONLY", + "raw_request": {"content": "TRANSIENT_CONTENT_ONLY"}, + } + ov_client = OpenVikingClient() + em_client = EverMemOSClient() + + ov_ingest = asyncio.run(ov_client.ingest_conversation_turn(payload)) + ov_commit = asyncio.run(ov_client.commit_session_v2(payload)) + em_ingest = em_client.ingest_message(payload) + em_commit = em_client.extract_profile_long_term_v2(payload) + + assert isinstance(ov_ingest, BackendWriteResult) + assert isinstance(em_ingest, BackendWriteResult) + assert isinstance(ov_commit, BackendCommitResult) + assert isinstance(em_commit, BackendCommitResult) + assert ov_ingest == ov_client._normalize_ingest_response( + { + "status": "skipped", + "session_id": "sess_contract", + "uri": "viking://sessions/sess_contract", + "metadata": { + "reason": "openviking_v2_ingest_adapter_not_configured", + "schema_version": "openviking.fixture.ingest.v2", + }, + } + ) + assert em_ingest == em_client._normalize_ingest_response( + { + "status": "skipped", + "memory_id": "turn_contract", + "metadata": { + "reason": "evermemos_v2_ingest_adapter_not_configured", + "schema_version": "evermemos.fixture.ingest.v2", + }, + } + ) + serialized = json.dumps( + { + "ov_ingest": ov_ingest.model_dump(mode="json"), + "ov_commit": ov_commit.model_dump(mode="json"), + "em_ingest": em_ingest.model_dump(mode="json"), + "em_commit": em_commit.model_dump(mode="json"), + }, + ensure_ascii=False, + ) + for blocked in ("TRANSIENT_CONTENT_ONLY", "content", "raw_request", "messages", "conversation", "transcript"): + assert blocked not in serialized + + +def test_retrieve_skeletons_use_retrieve_normalization_and_safe_metadata(): + payload = { + "workspace_id": "ws_1", + "user_id": "user_a", + "session_id": "sess_retrieve_contract", + "query": "fixture query", + "content": "TRANSIENT_RETRIEVE_CONTENT", + } + ov_result = asyncio.run(OpenVikingClient().retrieve_context_v2(payload)) + em_result = EverMemOSClient().retrieve_context_v2(payload) + + assert isinstance(ov_result, BackendRetrieveResult) + assert isinstance(em_result, BackendRetrieveResult) + assert ov_result.status == BackendResultStatus.SUCCESS + assert em_result.status == BackendResultStatus.SUCCESS + assert ov_result.items[0].source_backend == BackendType.OPENVIKING + assert em_result.items[0].source_backend == BackendType.EVERMEMOS + assert ov_result.items[0].text + assert em_result.items[0].ref_id + serialized = json.dumps( + {"ov": ov_result.model_dump(mode="json"), "em": em_result.model_dump(mode="json")}, + ensure_ascii=False, + ) + for blocked in ("TRANSIENT_RETRIEVE_CONTENT", "content", "raw_request", "messages", "conversation", "transcript"): + assert blocked not in serialized + + +def test_openviking_commit_skeleton_ref_type_is_mapped_from_fixture(): + result = asyncio.run(OpenVikingClient().commit_session_v2({"session_id": "sess_ov_map"})) + + assert result.refs + assert result.refs[0].ref_type == MemoryRefType.SESSION_ARCHIVE + assert result.refs[0].native_id == "ov_session_summary:sess_ov_map" + + +def test_evermemos_skeleton_multiple_refs_are_written_by_process_outbox_event(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SKIPPED) + ), + evermemos_client=EverMemOSClient(), + ) + response = asyncio.run( + service.commit_session("sess_em_skeleton", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = next(event for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.EVERMEMOS) + + updated = asyncio.run(service.process_outbox_event(event.id)) + refs = repo.list_memory_refs(session_id="sess_em_skeleton", backend_type=BackendType.EVERMEMOS, status=BackendRefStatus.SUCCESS) + + assert updated.status == OutboxEventStatus.SUCCESS + assert len(refs) == 2 + assert {ref.ref_type for ref in refs} == {MemoryRefType.PROFILE, MemoryRefType.LONG_TERM_MEMORY} + + +def test_obsidian_review_adapter_skeleton_returns_skipped_write_result(): + result = ObsidianReviewClient().create_review_draft_v2({"event_id": "evt_review"}) + + assert isinstance(result, BackendWriteResult) + assert result.backend_type == BackendType.OBSIDIAN + assert result.operation == BackendOperation.CREATE_REVIEW_DRAFT + assert result.status == BackendResultStatus.SKIPPED + + +def test_backend_commit_result_supports_multiple_produced_refs(): + result = BackendCommitResult( + backend_type=BackendType.EVERMEMOS, + status=BackendResultStatus.SUCCESS, + refs=[ + BackendProducedRef(ref_type=MemoryRefType.PROFILE, native_id="profile_1"), + BackendProducedRef(ref_type=MemoryRefType.LONG_TERM_MEMORY, native_uri="evermemos://memories/long_1"), + ], + ) + + dumped = result.model_dump(mode="json") + assert len(result.refs) == 2 + assert dumped["refs"][0]["ref_type"] == "profile" + assert dumped["refs"][1]["native_uri"] == "evermemos://memories/long_1" + + +def test_backend_ref_type_mapping_and_unknown_fallback_preserves_original_type(): + mapped, metadata = map_backend_ref_type(BackendType.OPENVIKING, "context_resource") + assert mapped == MemoryRefType.CONTEXT_RESOURCE + assert metadata == {} + + mapped, metadata = map_backend_ref_type(BackendType.OPENVIKING, "session_summary") + assert mapped == MemoryRefType.SESSION_ARCHIVE + assert metadata == {} + + mapped, metadata = map_backend_ref_type(BackendType.EVERMEMOS, "preference") + assert mapped == MemoryRefType.PROFILE + assert metadata == {} + + mapped, metadata = map_backend_ref_type(BackendType.EVERMEMOS, "unknown_signal") + assert mapped == MemoryRefType.LONG_TERM_MEMORY + assert metadata["original_ref_type"] == "unknown_signal" + + +def test_openviking_commit_fixture_normalizes_to_backend_commit_result_without_unsafe_metadata(): + raw = { + "status": "ok", + "session_id": "sess_norm", + "latency_ms": 18, + "metadata": {"backend_request_id": "ov_req_1", "content": "SECRET", "raw_request": {"content": "SECRET"}}, + "result": { + "refs": [ + { + "type": "session_archive", + "id": "ov_archive_1", + "uri": "viking://sessions/sess_norm", + "metadata": {"schema_version": "ov.v1", "messages": ["SECRET"]}, + } + ] + }, + } + + result = normalize_openviking_commit_response(raw) + + assert result.status == BackendResultStatus.SUCCESS + assert result.backend_type == BackendType.OPENVIKING + assert len(result.refs) == 1 + assert result.refs[0].ref_type == MemoryRefType.SESSION_ARCHIVE + assert result.refs[0].native_id == "ov_archive_1" + serialized = json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + assert "SECRET" not in serialized + assert "raw_request" not in serialized + assert "messages" not in serialized + + +def test_backend_response_fixture_files_exist_and_load(): + names = { + "openviking_ingest_success.json", + "openviking_ingest_real_success.json", + "openviking_ingest_real_error_401.json", + "openviking_ingest_real_error_422.json", + "openviking_ingest_real_error_500.json", + "openviking_commit_success.json", + "openviking_retrieve_success.json", + "evermemos_ingest_success.json", + "evermemos_commit_success_multiple_refs.json", + "evermemos_retrieve_success.json", + } + + for name in names: + payload = load_backend_fixture(name) + assert payload["status"] + + +def test_openviking_success_fixtures_normalize_without_unsafe_metadata(): + ingest = normalize_openviking_ingest_response(load_backend_fixture("openviking_ingest_success.json")) + commit = normalize_openviking_commit_response(load_backend_fixture("openviking_commit_success.json")) + retrieve = normalize_openviking_retrieve_response(load_backend_fixture("openviking_retrieve_success.json")) + + assert ingest.status == BackendResultStatus.SUCCESS + assert ingest.native_id == "ov_turn_fixture_1" + assert commit.status == BackendResultStatus.SUCCESS + assert {ref.ref_type for ref in commit.refs} == {MemoryRefType.SESSION_ARCHIVE, MemoryRefType.CONTEXT_RESOURCE} + assert retrieve.status == BackendResultStatus.SUCCESS + assert len(retrieve.items) == 2 + assert retrieve.items[0].source_backend == BackendType.OPENVIKING + serialized = json.dumps( + { + "ingest": ingest.model_dump(mode="json"), + "commit": commit.model_dump(mode="json"), + "retrieve": retrieve.model_dump(mode="json"), + }, + ensure_ascii=False, + ) + for blocked in ("content", "raw_request", "messages", "conversation", "transcript"): + assert blocked not in serialized + + +def test_evermemos_commit_fixture_normalizes_multiple_produced_refs_and_unknown_type(): + raw = { + "status": "success", + "data": { + "produced_refs": [ + {"ref_type": "episodic_memory", "memory_id": "episode_1", "metadata": {"confidence": 0.82}}, + {"ref_type": "profile", "profile_id": "profile_1", "metadata": {"content": "SECRET_PROFILE"}}, + {"ref_type": "unknown_kind", "id": "long_1", "metadata": {"score": 0.9}}, + ] + }, + } + + result = normalize_evermemos_commit_response(raw) + + assert result.status == BackendResultStatus.SUCCESS + assert len(result.refs) == 3 + assert [ref.ref_type for ref in result.refs] == [ + MemoryRefType.EPISODIC_MEMORY, + MemoryRefType.PROFILE, + MemoryRefType.LONG_TERM_MEMORY, + ] + assert result.refs[2].metadata["original_ref_type"] == "unknown_kind" + assert "SECRET_PROFILE" not in json.dumps(result.model_dump(mode="json"), ensure_ascii=False) + + +def test_evermemos_success_fixtures_normalize_without_unsafe_metadata(): + ingest = normalize_evermemos_ingest_response(load_backend_fixture("evermemos_ingest_success.json")) + commit = normalize_evermemos_commit_response(load_backend_fixture("evermemos_commit_success_multiple_refs.json")) + retrieve = normalize_evermemos_retrieve_response(load_backend_fixture("evermemos_retrieve_success.json")) + + assert ingest.status == BackendResultStatus.SUCCESS + assert ingest.native_id == "em_memory_fixture_1" + assert commit.status == BackendResultStatus.SUCCESS + assert {ref.ref_type for ref in commit.refs} == { + MemoryRefType.EPISODIC_MEMORY, + MemoryRefType.PROFILE, + MemoryRefType.LONG_TERM_MEMORY, + } + assert retrieve.status == BackendResultStatus.SUCCESS + assert len(retrieve.items) == 2 + assert retrieve.items[0].source_backend == BackendType.EVERMEMOS + assert retrieve.items[0].memory_type == "episodic_memory" + serialized = json.dumps( + { + "ingest": ingest.model_dump(mode="json"), + "commit": commit.model_dump(mode="json"), + "retrieve": retrieve.model_dump(mode="json"), + }, + ensure_ascii=False, + ) + for blocked in ("content", "raw_request", "messages", "conversation", "transcript"): + assert blocked not in serialized + + +def test_malformed_retrieve_response_returns_skipped_empty_result(): + ov = normalize_openviking_retrieve_response({}) + em = normalize_evermemos_retrieve_response({"data": {"unexpected": "shape"}}) + + assert ov.status == BackendResultStatus.SKIPPED + assert ov.items == [] + assert em.status == BackendResultStatus.SUCCESS + assert em.items == [] + + +def test_ingest_response_normalizers_return_write_results_and_sanitize_metadata(): + ov = normalize_openviking_ingest_response( + { + "status": "created", + "id": "ov_turn_1", + "uri": "viking://sessions/sess/turn", + "metadata": {"backend_request_id": "ov_req", "conversation": "SECRET"}, + } + ) + em = normalize_evermemos_ingest_response( + { + "status": "success", + "memory_id": "em_turn_1", + "metadata": {"trace_id": "trace_1", "transcript": "SECRET"}, + } + ) + + assert isinstance(ov, BackendWriteResult) + assert isinstance(em, BackendWriteResult) + assert ov.native_id == "ov_turn_1" + assert em.native_id == "em_turn_1" + serialized = json.dumps({"ov": ov.model_dump(mode="json"), "em": em.model_dump(mode="json")}, ensure_ascii=False) + assert "SECRET" not in serialized + assert "conversation" not in serialized + assert "transcript" not in serialized + + +def test_backend_error_retryable_mapping(): + for status_code in (429, 500, 502, 503, 504): + assert map_backend_error_to_retryable(BackendType.OPENVIKING, status_code=status_code) is True + assert map_backend_error_to_retryable(BackendType.EVERMEMOS, error_code="timeout") is True + assert map_backend_error_to_retryable(BackendType.EVERMEMOS, error_message="network_error: reset") is True + assert map_backend_error_to_retryable(BackendType.OPENVIKING, error_code="mystery") is True + + for status_code in (400, 401, 403, 404, 422): + assert map_backend_error_to_retryable(BackendType.EVERMEMOS, status_code=status_code) is False + + +def test_client_map_error_contracts_for_future_http_integration(): + class ResponseLike: + def __init__(self, status_code): + self.status_code = status_code + + def __str__(self): + return f"response {self.status_code}" + + ov_client = OpenVikingClient() + em_client = EverMemOSClient() + + for status_code in (429, 500, 502, 503, 504): + assert ov_client._map_error(ResponseLike(status_code)) is True + assert em_client._map_error(ResponseLike(status_code)) is True + for status_code in (400, 401, 403, 404, 422): + assert ov_client._map_error(ResponseLike(status_code)) is False + assert em_client._map_error(ResponseLike(status_code)) is False + + assert ov_client._map_error(TimeoutError("timeout while reading")) is True + assert em_client._map_error(ConnectionError("network_error connection reset")) is True + assert ov_client._map_error(RuntimeError("unknown backend failure")) is True + + +def test_v2_ingest_schema_constructs(): + request = IngestRequest(**build_ingest_payload()) + + assert request.workspace_id == "ws_1" + assert request.request_id == "req_1" + assert request.policy.allow_openviking is True + + +def test_ingest_service_records_two_success_refs(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + + response = asyncio.run(service.ingest_conversation_turn(IngestRequest(**build_ingest_payload()))) + + assert response.status == "success" + assert len(response.refs) == 2 + assert {ref.backend_type.value for ref in response.refs} == {"openviking", "evermemos"} + assert {ref.status for ref in repo.list_memory_refs()} == {BackendRefStatus.SUCCESS} + assert len(repo.list_memory_refs(backend_type="openviking", status=BackendRefStatus.SUCCESS)) == 1 + + +def test_v2_ingest_service_openviking_real_mock_success_writes_safe_memory_ref(): + fixture = load_backend_fixture("openviking_ingest_real_success.json") + + def handler(request): + payload = json.loads(request.content.decode()) + assert payload["content"] == "SECRET_SERVICE_REAL_CONTENT" + assert request.headers["x-api-key"] == "ov-super-secret-token" + return httpx.Response(200, json=fixture) + + async def real_openviking_factory(): + return OpenVikingClient( + mode="real", + base_url="http://openviking.test", + api_key="ov-super-secret-token", + transport=httpx.MockTransport(handler), + ) + + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=real_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + + response = asyncio.run( + service.ingest_conversation_turn( + IngestRequest(**build_ingest_payload(session_id="ov_real_sess_fixture_1", content="SECRET_SERVICE_REAL_CONTENT")) + ) + ) + ov_ref = repo.list_memory_refs(backend_type=BackendType.OPENVIKING, status=BackendRefStatus.SUCCESS)[0] + audit_json = json.dumps([entry.model_dump(mode="json") for entry in repo.list_audit()], ensure_ascii=False) + + assert response.status == OperationStatus.SUCCESS + assert ov_ref.native_id == "ov_real_turn_fixture_1" + assert ov_ref.native_uri == "viking://sessions/ov_real_sess_fixture_1/turns/ov_real_turn_fixture_1" + serialized = json.dumps(ov_ref.model_dump(mode="json"), ensure_ascii=False) + assert "SECRET_SERVICE_REAL_CONTENT" not in serialized + assert "ov-super-secret-token" not in serialized + assert "raw_request" not in serialized + assert "content" not in ov_ref.metadata + assert "ov-super-secret-token" not in audit_json + + +def test_v2_ingest_service_real_mock_success_writes_openviking_and_evermemos_refs_safely(): + ov_fixture = load_backend_fixture("openviking_ingest_real_success.json") + em_fixture = load_backend_fixture("evermemos_ingest_success.json") + seen = {"openviking": 0, "evermemos": 0} + + def openviking_handler(request): + payload = json.loads(request.content.decode()) + assert payload["content"] == "SECRET_DUAL_REAL_CONTENT" + assert request.headers["x-api-key"] == "ov-dual-token" + seen["openviking"] += 1 + return httpx.Response(200, json=ov_fixture) + + def evermemos_handler(request): + payload = json.loads(request.content.decode()) + assert payload["content"] == "SECRET_DUAL_REAL_CONTENT" + assert request.headers["x-api-key"] == "em-dual-token" + assert request.headers["authorization"] == "Bearer em-dual-token" + seen["evermemos"] += 1 + return httpx.Response(200, json=em_fixture) + + async def real_openviking_factory(): + return OpenVikingClient( + mode="real", + base_url="http://openviking.test", + api_key="ov-dual-token", + transport=httpx.MockTransport(openviking_handler), + ) + + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=real_openviking_factory, + evermemos_client=EverMemOSClient( + mode="real", + base_url="http://evermemos.test", + api_key="em-dual-token", + transport=httpx.MockTransport(evermemos_handler), + ), + ) + + response = asyncio.run( + service.ingest_conversation_turn( + IngestRequest( + **build_ingest_payload( + session_id="ov_real_sess_fixture_1", + source_type="cli", + content="SECRET_DUAL_REAL_CONTENT", + trace={"trace_id": "trace_dual_real", "request_id": "trace_req_dual"}, + ) + ) + ) + ) + + refs = repo.list_memory_refs() + serialized_refs = json.dumps([ref.model_dump(mode="json") for ref in refs], ensure_ascii=False) + audit_json = json.dumps([entry.model_dump(mode="json") for entry in repo.list_audit()], ensure_ascii=False) + + assert response.status == OperationStatus.SUCCESS + assert seen == {"openviking": 1, "evermemos": 1} + assert {ref.backend_type for ref in refs} == {BackendType.OPENVIKING, BackendType.EVERMEMOS} + assert {ref.status for ref in refs} == {BackendRefStatus.SUCCESS} + assert {ref.content_hash for ref in refs} + assert "trace_dual_real" in serialized_refs + for blocked in ("SECRET_DUAL_REAL_CONTENT", "ov-dual-token", "em-dual-token", "raw_request", "messages", "conversation", "transcript"): + assert blocked not in serialized_refs + for blocked in ("SECRET_DUAL_REAL_CONTENT", "ov-dual-token", "em-dual-token", "raw_request", "messages", "transcript"): + assert blocked not in audit_json + + +def test_ingest_service_backend_failure_is_partial_success(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_openviking_factory, + evermemos_client=FailingEverMemOSClient(), + ) + + response = asyncio.run(service.ingest_conversation_turn(IngestRequest(**build_ingest_payload()))) + + assert response.status == "partial_success" + assert len(response.refs) == 2 + failed = [ref for ref in response.refs if ref.status == BackendRefStatus.FAILED] + assert len(failed) == 1 + assert failed[0].backend_type.value == "evermemos" + assert "evermemos unavailable" in failed[0].error_message + + +def test_ingest_service_records_two_skipped_refs_when_policy_disables_backends(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + + response = asyncio.run( + service.ingest_conversation_turn( + IngestRequest( + **build_ingest_payload( + policy={ + "allow_openviking": False, + "allow_evermemos": False, + } + ) + ) + ) + ) + + assert response.status == "skipped" + assert len(response.refs) == 2 + assert {ref.status for ref in response.refs} == {BackendRefStatus.SKIPPED} + assert len(repo.list_memory_refs()) == 2 + + +def test_duplicate_idempotency_key_upserts_memory_refs_without_duplicates(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + + first = asyncio.run( + service.ingest_conversation_turn( + IngestRequest(**build_ingest_payload(idempotency_key="idem_1", request_id="req_1")) + ) + ) + second = asyncio.run( + service.ingest_conversation_turn( + IngestRequest( + **build_ingest_payload( + idempotency_key="idem_1", + request_id="req_2", + source_event_id="evt_changed", + turn_id="turn_changed", + ) + ) + ) + ) + + refs = repo.list_memory_refs() + assert len(refs) == 2 + assert {ref.id for ref in first.refs} == {ref.id for ref in second.refs} + assert first.gateway_id == second.gateway_id + + +def test_memory_ref_metadata_does_not_store_conversation_content_or_raw_request(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + sensitive_content = "SECRET_CONVERSATION_CONTENT_SHOULD_NOT_BE_STORED" + + asyncio.run( + service.ingest_conversation_turn( + IngestRequest( + **build_ingest_payload( + content=sensitive_content, + metadata={"channel": "cli", "raw_request": {"content": sensitive_content}}, + ) + ) + ) + ) + + for ref in repo.list_memory_refs(): + metadata_json = json.dumps(ref.metadata, ensure_ascii=False) + assert sensitive_content not in metadata_json + assert "raw_request" not in metadata_json + assert ref.content_hash + assert ref.content_hash in metadata_json + + +def test_sqlite_repository_persists_v2_memory_refs(tmp_path): + repo = SQLiteRepository(tmp_path / "memory_gateway.sqlite3") + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + + asyncio.run(service.ingest_conversation_turn(IngestRequest(**build_ingest_payload(turn_id="turn_sqlite")))) + + reloaded = SQLiteRepository(tmp_path / "memory_gateway.sqlite3") + refs = reloaded.list_memory_refs( + workspace_id="ws_1", + backend_type="openviking", + status=BackendRefStatus.SUCCESS, + ) + assert len(refs) == 1 + assert refs[0].turn_id == "turn_sqlite" + + +def test_commit_session_creates_commit_job_and_outbox_events(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service(repo=repo) + + response = asyncio.run( + service.commit_session( + "sess_commit", + CommitRequest( + workspace_id="ws_1", + user_id="user_a", + agent_id="agent_cli", + namespace="workspace/ws_1/user/user_a", + request_id="commit_req_1", + ), + ) + ) + + job = repo.get_commit_job(response.job_id) + events = repo.list_outbox_events(gateway_id=response.metadata["gateway_id"]) + + assert response.status == "accepted" + assert job is not None + assert job.session_id == "sess_commit" + assert job.status.value == "accepted" + assert len(events) == 2 + assert {event.backend_type for event in events} == {BackendType.OPENVIKING, BackendType.EVERMEMOS} + assert {event.operation for event in events} == {BackendOperation.COMMIT_SESSION} + assert {event.status for event in events} == {OutboxEventStatus.PENDING} + + +def test_sqlite_repository_persists_commit_job_and_outbox_events(tmp_path): + repo = SQLiteRepository(tmp_path / "memory_gateway.sqlite3") + service = MemoryGatewayV2Service(repo=repo) + + response = asyncio.run( + service.commit_session( + "sess_commit_sqlite", + CommitRequest( + workspace_id="ws_1", + user_id="user_a", + agent_id="agent_cli", + namespace="workspace/ws_1/user/user_a", + idempotency_key="commit_idem_1", + ), + ) + ) + + reloaded = SQLiteRepository(tmp_path / "memory_gateway.sqlite3") + job = reloaded.get_commit_job(response.job_id) + events = reloaded.list_outbox_events(gateway_id=response.metadata["gateway_id"]) + + assert job is not None + assert job.session_id == "sess_commit_sqlite" + assert len(events) == 2 + assert {event.payload_ref for event in events} == {f"commit_job:{response.job_id}"} + + +def test_sqlite_repository_claims_due_outbox_with_lease_fields(tmp_path): + repo = SQLiteRepository(tmp_path / "memory_gateway.sqlite3") + service = MemoryGatewayV2Service(repo=repo) + response = asyncio.run( + service.commit_session( + "sess_sqlite_claim", + CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli"), + ) + ) + + claimed = repo.claim_pending_outbox_events(limit=1, worker_id="sqlite_worker", lease_seconds=30) + reloaded = SQLiteRepository(tmp_path / "memory_gateway.sqlite3") + events = reloaded.list_outbox_events_by_job(response.job_id) + + assert len(claimed) == 1 + assert sum(1 for event in events if event.status == OutboxEventStatus.PROCESSING) == 1 + claimed_event = next(event for event in events if event.status == OutboxEventStatus.PROCESSING) + assert claimed_event.locked_by == "sqlite_worker" + assert claimed_event.lease_expires_at is not None + + +def test_outbox_event_does_not_store_conversation_content_or_raw_request(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service(repo=repo) + sensitive_content = "SECRET_COMMIT_CONTENT_SHOULD_NOT_BE_STORED" + + response = asyncio.run( + service.commit_session( + "sess_commit", + CommitRequest( + workspace_id="ws_1", + user_id="user_a", + agent_id="agent_cli", + namespace="workspace/ws_1/user/user_a", + metadata={"raw_request": {"content": sensitive_content}}, + ), + ) + ) + + for event in repo.list_outbox_events(gateway_id=response.metadata["gateway_id"]): + event_json = json.dumps(event.model_dump(mode="json"), ensure_ascii=False) + assert sensitive_content not in event_json + assert "raw_request" not in event_json + assert event.payload_ref == f"commit_job:{response.job_id}" + + +def test_retrieve_response_contract_contains_items_refs_conflicts_trace_id_status(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + asyncio.run(service.ingest_conversation_turn(IngestRequest(**build_ingest_payload()))) + + response = asyncio.run( + service.retrieve_context( + RetrieveRequest( + workspace_id="ws_1", + user_id="user_a", + agent_id="agent_cli", + session_id="sess_1", + query="remember", + metadata={"trace_id": "trace_1"}, + ) + ) + ) + + dumped = response.model_dump() + assert set(["items", "refs", "conflicts", "trace_id", "status"]).issubset(dumped) + assert response.trace_id == "trace_1" + assert response.status.value == "success" + assert len(response.items) == len(response.refs) + assert response.conflicts == [] + + +def test_process_commit_job_success_updates_job_and_writes_memory_refs(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_commit_1", native_uri="viking://sessions/sess_commit") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.SUCCESS, native_id="em_commit_1", native_uri="evermemos://memories/em_commit_1") + ), + ) + response = asyncio.run( + service.commit_session( + "sess_commit", + CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli", namespace="workspace/ws_1/user/user_a"), + ) + ) + + job = asyncio.run(service.process_commit_job(response.job_id)) + events = repo.list_outbox_events_by_job(response.job_id) + refs = repo.list_memory_refs(session_id="sess_commit", status=BackendRefStatus.SUCCESS) + + assert job.status.value == "success" + assert job.started_at is not None + assert job.finished_at is not None + assert job.created_refs_count == 2 + assert {event.status for event in events} == {OutboxEventStatus.SUCCESS} + assert len(refs) == 2 + assert {ref.backend_type for ref in refs} == {BackendType.OPENVIKING, BackendType.EVERMEMOS} + + +def test_process_outbox_event_writes_multiple_produced_memory_refs(): + repo = InMemoryRepository() + sensitive_content = "SECRET_PRODUCED_REF_CONTENT" + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + BackendCommitResult( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.COMMIT_SESSION, + status=BackendResultStatus.SUCCESS, + refs=[ + BackendProducedRef( + ref_type=MemoryRefType.SESSION_ARCHIVE, + native_id="ov_session_archive_1", + native_uri="viking://sessions/sess_multi", + metadata={"backend_request_id": "req_ov_1", "content": sensitive_content}, + ), + BackendProducedRef( + ref_type=MemoryRefType.PROFILE, + native_id="ov_profile_1", + metadata={"source_channel": "worker", "raw_request": {"content": sensitive_content}}, + ), + ], + metadata={"latency_ms": 12, "messages": [sensitive_content]}, + ) + ), + ) + response = asyncio.run( + service.commit_session( + "sess_multi", + CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli", namespace="workspace/ws_1/user/user_a"), + ) + ) + event = next(event for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.OPENVIKING) + + updated = asyncio.run(service.process_outbox_event(event.id)) + refs = repo.list_memory_refs(session_id="sess_multi", backend_type=BackendType.OPENVIKING, status=BackendRefStatus.SUCCESS) + + assert updated.status == OutboxEventStatus.SUCCESS + assert len(refs) == 2 + assert {ref.ref_type for ref in refs} == {MemoryRefType.SESSION_ARCHIVE, MemoryRefType.PROFILE} + assert {ref.native_id for ref in refs} == {"ov_session_archive_1", "ov_profile_1"} + for ref in refs: + serialized = json.dumps(ref.model_dump(mode="json"), ensure_ascii=False) + assert sensitive_content not in serialized + assert "raw_request" not in serialized + assert "messages" not in serialized + assert "conversation" not in serialized + assert "transcript" not in serialized + + +def test_process_outbox_event_writes_same_ref_type_with_different_native_ids(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + BackendCommitResult( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.COMMIT_SESSION, + status=BackendResultStatus.SUCCESS, + refs=[ + BackendProducedRef(ref_type=MemoryRefType.CONTEXT_RESOURCE, native_id="resource_1"), + BackendProducedRef(ref_type=MemoryRefType.CONTEXT_RESOURCE, native_id="resource_2"), + ], + ) + ), + ) + response = asyncio.run( + service.commit_session("sess_same_type", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = next(event for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.OPENVIKING) + + asyncio.run(service.process_outbox_event(event.id)) + refs = repo.list_memory_refs(session_id="sess_same_type", backend_type=BackendType.OPENVIKING, ref_type=MemoryRefType.CONTEXT_RESOURCE) + + assert len(refs) == 2 + assert {ref.native_id for ref in refs} == {"resource_1", "resource_2"} + assert len({ref.id for ref in refs}) == 2 + + +def test_memory_ref_id_uses_stable_fallback_when_native_ref_is_missing(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + BackendCommitResult( + backend_type=BackendType.OPENVIKING, + operation=BackendOperation.COMMIT_SESSION, + status=BackendResultStatus.SUCCESS, + refs=[ + BackendProducedRef(ref_type=MemoryRefType.SESSION_ARCHIVE, metadata={"stable_key": "summary_a"}), + BackendProducedRef(ref_type=MemoryRefType.SESSION_ARCHIVE, metadata={"stable_key": "summary_b"}), + ], + ) + ), + ) + response = asyncio.run( + service.commit_session("sess_stable_key", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = next(event for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.OPENVIKING) + + asyncio.run(service.process_outbox_event(event.id)) + refs = repo.list_memory_refs(session_id="sess_stable_key", backend_type=BackendType.OPENVIKING, ref_type=MemoryRefType.SESSION_ARCHIVE) + + assert len(refs) == 2 + assert len({ref.id for ref in refs}) == 2 + assert {ref.metadata["stable_key"] for ref in refs} == {"summary_a", "summary_b"} + + +def test_process_outbox_event_keeps_single_native_ref_fallback_compatible(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_single", native_uri="viking://sessions/ov_single") + ), + ) + response = asyncio.run( + service.commit_session("sess_single_fallback", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = next(event for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.OPENVIKING) + + asyncio.run(service.process_outbox_event(event.id)) + refs = repo.list_memory_refs(session_id="sess_single_fallback", backend_type=BackendType.OPENVIKING, status=BackendRefStatus.SUCCESS) + + assert len(refs) == 1 + assert refs[0].ref_type == MemoryRefType.SESSION_ARCHIVE + assert refs[0].native_id == "ov_single" + + +def test_process_commit_job_one_success_one_failed_is_partial_success(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_commit_1") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.FAILED, retryable=False, error_message="evermemos failed") + ), + ) + response = asyncio.run( + service.commit_session("sess_partial", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + + job = asyncio.run(service.process_commit_job(response.job_id)) + events = repo.list_outbox_events_by_job(response.job_id) + + assert job.status.value == "partial_success" + assert job.created_refs_count == 1 + assert "evermemos failed" in job.error_message + assert {event.status for event in events} == {OutboxEventStatus.SUCCESS, OutboxEventStatus.DEAD_LETTER} + + +def test_process_commit_job_two_failed_is_failed(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.FAILED, retryable=False, error_message="openviking failed") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.FAILED, retryable=False, error_message="evermemos failed") + ), + ) + response = asyncio.run( + service.commit_session("sess_failed", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + + job = asyncio.run(service.process_commit_job(response.job_id)) + + assert job.status.value == "failed" + assert job.created_refs_count == 0 + assert "openviking failed" in job.error_message + assert "evermemos failed" in job.error_message + + +def test_retryable_failed_outbox_event_requeues_with_next_retry(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.FAILED, retryable=True, error_message="temporary openviking failure") + ), + ) + response = asyncio.run( + service.commit_session("sess_retry", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = next(event for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.OPENVIKING) + + updated = asyncio.run(service.process_outbox_event(event.id)) + + assert updated.status == OutboxEventStatus.PENDING + assert updated.attempt_count == 1 + assert updated.next_retry_at is not None + assert "temporary openviking failure" in updated.last_error + + +def test_process_pending_outbox_events_processes_pending_batch(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_commit_1") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.SUCCESS, native_id="em_commit_1") + ), + ) + asyncio.run( + service.commit_session("sess_batch", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + + processed = asyncio.run(service.process_pending_outbox_events()) + + assert len(processed) == 2 + assert {event.status for event in processed} == {OutboxEventStatus.SUCCESS} + assert len(repo.list_memory_refs(session_id="sess_batch", status=BackendRefStatus.SUCCESS)) == 2 + + +def test_retryable_failed_outbox_event_exceeding_max_attempts_dead_letters(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.FAILED, retryable=True, error_message="still failing") + ), + ) + response = asyncio.run( + service.commit_session("sess_dead", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = next(event for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.OPENVIKING) + event.max_attempts = 1 + repo.save_outbox_event(event) + + updated = asyncio.run(service.process_outbox_event(event.id)) + + assert updated.status == OutboxEventStatus.DEAD_LETTER + assert updated.attempt_count == 1 + assert updated.next_retry_at is None + + +def test_commit_pipeline_metadata_does_not_store_content_or_raw_request(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_commit_1") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.SUCCESS, native_id="em_commit_1") + ), + ) + sensitive_content = "SECRET_COMMIT_PIPELINE_CONTENT_SHOULD_NOT_BE_STORED" + response = asyncio.run( + service.commit_session( + "sess_secure", + CommitRequest( + workspace_id="ws_1", + user_id="user_a", + agent_id="agent_cli", + metadata={"raw_request": {"content": sensitive_content}}, + ), + ) + ) + + asyncio.run(service.process_commit_job(response.job_id)) + + for event in repo.list_outbox_events_by_job(response.job_id): + assert sensitive_content not in json.dumps(event.model_dump(mode="json"), ensure_ascii=False) + assert "raw_request" not in json.dumps(event.model_dump(mode="json"), ensure_ascii=False) + for ref in repo.list_memory_refs(session_id="sess_secure"): + assert sensitive_content not in json.dumps(ref.model_dump(mode="json"), ensure_ascii=False) + assert "raw_request" not in json.dumps(ref.model_dump(mode="json"), ensure_ascii=False) + + +def test_claim_pending_outbox_events_only_claims_due_pending_events(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service(repo=repo) + response = asyncio.run( + service.commit_session("sess_claim", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + events = repo.list_outbox_events_by_job(response.job_id) + delayed = events[0] + delayed.next_retry_at = datetime.now(timezone.utc) + timedelta(minutes=5) + repo.save_outbox_event(delayed) + + claimed = repo.claim_pending_outbox_events(limit=10, worker_id="worker_claim", lease_seconds=30) + + assert len(claimed) == 1 + assert claimed[0].id != delayed.id + assert claimed[0].status == OutboxEventStatus.PROCESSING + assert claimed[0].locked_by == "worker_claim" + assert claimed[0].lease_expires_at is not None + + +def test_next_retry_not_due_event_is_not_claimed(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service(repo=repo) + response = asyncio.run( + service.commit_session("sess_retry_wait", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + for event in repo.list_outbox_events_by_job(response.job_id): + event.next_retry_at = datetime.now(timezone.utc) + timedelta(minutes=5) + repo.save_outbox_event(event) + + claimed = repo.claim_pending_outbox_events(limit=10, worker_id="worker_wait", lease_seconds=30) + + assert claimed == [] + assert {event.status for event in repo.list_outbox_events_by_job(response.job_id)} == {OutboxEventStatus.PENDING} + + +def test_expired_processing_event_is_released_to_pending(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service(repo=repo) + response = asyncio.run( + service.commit_session("sess_expired", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + claimed = repo.claim_pending_outbox_events(limit=1, worker_id="worker_old", lease_seconds=1) + assert len(claimed) == 1 + + released = repo.release_expired_processing_events(datetime.now(timezone.utc) + timedelta(seconds=2)) + + assert len(released) == 1 + assert released[0].status == OutboxEventStatus.PENDING + assert released[0].locked_by is None + assert released[0].lease_expires_at is None + + +def test_process_pending_outbox_events_uses_claim_and_does_not_process_existing_lock(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_claimed") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.SUCCESS, native_id="em_claimed") + ), + ) + response = asyncio.run( + service.commit_session("sess_no_double", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + externally_claimed = repo.claim_pending_outbox_events(limit=1, worker_id="worker_a", lease_seconds=300)[0] + + processed = asyncio.run(service.process_pending_outbox_events(worker_id="worker_b")) + events = repo.list_outbox_events_by_job(response.job_id) + + assert len(processed) == 1 + assert sum(1 for event in events if event.status == OutboxEventStatus.SUCCESS) == 1 + still_locked = next(event for event in events if event.id == externally_claimed.id) + assert still_locked.status == OutboxEventStatus.PROCESSING + assert still_locked.locked_by == "worker_a" + + +def test_terminal_outbox_statuses_clear_lock_fields(): + repo = InMemoryRepository() + success_service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_lock_clear") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.SKIPPED) + ), + ) + response = asyncio.run( + success_service.commit_session("sess_lock_clear", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + + processed = asyncio.run(success_service.process_pending_outbox_events(worker_id="worker_lock")) + + assert {event.status for event in processed} == {OutboxEventStatus.SUCCESS, OutboxEventStatus.SKIPPED} + assert all(event.locked_by is None for event in processed) + assert all(event.lease_expires_at is None for event in processed) + + fail_service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.FAILED, retryable=False, error_message="fatal") + ), + ) + failed = asyncio.run( + fail_service.commit_session("sess_dead_lock", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + event = next(event for event in repo.list_outbox_events_by_job(failed.job_id) if event.backend_type == BackendType.OPENVIKING) + updated = asyncio.run(fail_service.process_outbox_event(event.id)) + + assert updated.status == OutboxEventStatus.DEAD_LETTER + assert updated.locked_by is None + assert updated.lease_expires_at is None + assert repo.list_outbox_events_by_job(response.job_id) + + +def test_retryable_failed_outbox_event_clears_lock_when_requeued(): + repo = InMemoryRepository() + service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.FAILED, retryable=True, error_message="temporary") + ), + ) + response = asyncio.run( + service.commit_session("sess_retry_lock", CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli")) + ) + + updated = asyncio.run( + service.process_outbox_event( + next(event.id for event in repo.list_outbox_events_by_job(response.job_id) if event.backend_type == BackendType.OPENVIKING) + ) + ) + + assert updated.status == OutboxEventStatus.PENDING + assert updated.next_retry_at is not None + assert updated.locked_by is None + assert updated.lease_expires_at is None + + +def test_job_query_api_returns_job_status_and_outbox_summary(monkeypatch): + import memory_gateway.api_v2 as api_v2 + + repo = InMemoryRepository() + api_v2.v2_service = MemoryGatewayV2Service(repo=repo) + commit_response = asyncio.run( + api_v2.v2_service.commit_session( + "sess_job_api", + CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli", namespace="workspace/ws_1/user/user_a"), + ) + ) + app = FastAPI() + app.dependency_overrides[verify_api_key_compat] = lambda: None + app.include_router(api_v2.router) + + async def get_request(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + return await client.get(f"/v2/jobs/{commit_response.job_id}") + + response = asyncio.run(asyncio.wait_for(get_request(), timeout=2)) + + assert response.status_code == 200 + payload = response.json() + assert payload["job_id"] == commit_response.job_id + assert payload["status"] == "accepted" + assert payload["outbox_summary"]["total_events"] == 2 + assert payload["outbox_summary"]["pending_events"] == 2 + + +def test_admin_process_outbox_endpoint_triggers_pending_processing(monkeypatch): + import memory_gateway.api_v2 as api_v2 + + repo = InMemoryRepository() + api_v2.v2_service = MemoryGatewayV2Service( + repo=repo, + openviking_client_factory=fake_commit_openviking_factory( + commit_result(BackendType.OPENVIKING, BackendResultStatus.SUCCESS, native_id="ov_admin") + ), + evermemos_client=FakeCommitEverMemOSClient( + commit_result(BackendType.EVERMEMOS, BackendResultStatus.SUCCESS, native_id="em_admin") + ), + ) + asyncio.run( + api_v2.v2_service.commit_session( + "sess_admin", + CommitRequest(workspace_id="ws_1", user_id="user_a", agent_id="agent_cli"), + ) + ) + app = FastAPI() + app.dependency_overrides[verify_api_key_compat] = lambda: None + app.include_router(api_v2.router) + + async def post_request(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + return await client.post("/v2/admin/outbox/process?limit=10&worker_id=test_worker") + + response = asyncio.run(asyncio.wait_for(post_request(), timeout=2)) + + assert response.status_code == 200 + payload = response.json() + assert payload["worker_id"] == "test_worker" + assert payload["processed_count"] == 2 + assert payload["outbox_summary"]["success_events"] == 2 + + +def test_worker_v2_cli_processes_once_and_prints_control_plane_summary(monkeypatch, capsys): + import memory_gateway.worker_v2 as worker_v2 + + class FakeWorkerService: + async def process_pending_outbox_events_summary(self, limit: int, worker_id: str, lease_seconds: int): + assert limit == 7 + assert worker_id == "cli_worker" + assert lease_seconds == 45 + return OutboxProcessResponse( + status=OperationStatus.SUCCESS, + worker_id=worker_id, + processed_count=2, + ) + + monkeypatch.setattr(worker_v2, "v2_service", FakeWorkerService()) + + exit_code = worker_v2.main(["--limit", "7", "--worker-id", "cli_worker", "--lease-seconds", "45"]) + + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["worker_id"] == "cli_worker" + assert payload["processed_count"] == 2 + assert "content" not in json.dumps(payload) + assert "raw_request" not in json.dumps(payload) + + +def test_v2_ingest_router_accepts_legal_request(monkeypatch): + import memory_gateway.api_v2 as api_v2 + + api_v2.v2_service = MemoryGatewayV2Service( + repo=InMemoryRepository(), + openviking_client_factory=fake_openviking_factory, + evermemos_client=FakeEverMemOSClient(), + ) + app = FastAPI() + app.dependency_overrides[verify_api_key_compat] = lambda: None + app.include_router(api_v2.router) + + async def post_request(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + return await client.post("/v2/conversations/ingest", json=build_ingest_payload(turn_id="turn_router")) + + response = asyncio.run(asyncio.wait_for(post_request(), timeout=2)) + + assert response.status_code == 200 + payload = response.json() + assert payload["turn_id"] == "turn_router" + assert len(payload["refs"]) == 2