Refactor code structure for improved readability and maintainability
This commit is contained in:
259
memory_gateway/backend_normalization.py
Normal file
259
memory_gateway/backend_normalization.py
Normal file
@ -0,0 +1,259 @@
|
||||
"""Offline response normalization helpers for future v2 backend adapters."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .backend_contracts import (
|
||||
BackendCommitResult,
|
||||
BackendOperation,
|
||||
BackendProducedRef,
|
||||
BackendResultStatus,
|
||||
BackendRetrieveItem,
|
||||
BackendRetrieveResult,
|
||||
BackendWriteResult,
|
||||
)
|
||||
from .backend_ref_mapping import map_backend_ref_type
|
||||
from .schemas_v2 import BackendType
|
||||
|
||||
|
||||
SAFE_METADATA_KEYS = {
|
||||
"backend_request_id",
|
||||
"request_id",
|
||||
"trace_id",
|
||||
"latency_ms",
|
||||
"schema_version",
|
||||
"source_channel",
|
||||
"reason",
|
||||
"original_ref_type",
|
||||
"confidence",
|
||||
"score",
|
||||
"version",
|
||||
"created_at",
|
||||
}
|
||||
BLOCKED_METADATA_KEYS = {"content", "raw_request", "messages", "conversation", "transcript"}
|
||||
|
||||
|
||||
def safe_backend_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not metadata:
|
||||
return {}
|
||||
safe: dict[str, Any] = {}
|
||||
for key, value in metadata.items():
|
||||
if key in BLOCKED_METADATA_KEYS or key not in SAFE_METADATA_KEYS:
|
||||
continue
|
||||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||||
safe[key] = value
|
||||
return safe
|
||||
|
||||
|
||||
def normalize_openviking_commit_response(raw: dict[str, Any]) -> BackendCommitResult:
|
||||
status = _result_status(raw)
|
||||
refs = [_produced_ref(BackendType.OPENVIKING, item) for item in _extract_ref_items(raw)]
|
||||
return BackendCommitResult(
|
||||
backend_type=BackendType.OPENVIKING,
|
||||
operation=BackendOperation.COMMIT_SESSION,
|
||||
status=status,
|
||||
native_id=raw.get("native_id") or raw.get("session_id"),
|
||||
native_uri=raw.get("native_uri") or raw.get("uri"),
|
||||
retryable=_retryable_from_raw(BackendType.OPENVIKING, raw),
|
||||
error_code=raw.get("error_code"),
|
||||
error_message=raw.get("error") or raw.get("error_message"),
|
||||
latency_ms=raw.get("latency_ms"),
|
||||
refs=refs,
|
||||
metadata=safe_backend_metadata(raw.get("metadata") or raw),
|
||||
)
|
||||
|
||||
|
||||
def normalize_evermemos_commit_response(raw: dict[str, Any]) -> BackendCommitResult:
|
||||
status = _result_status(raw)
|
||||
refs = [_produced_ref(BackendType.EVERMEMOS, item) for item in _extract_ref_items(raw)]
|
||||
return BackendCommitResult(
|
||||
backend_type=BackendType.EVERMEMOS,
|
||||
operation=BackendOperation.COMMIT_SESSION,
|
||||
status=status,
|
||||
native_id=raw.get("native_id") or raw.get("session_id"),
|
||||
native_uri=raw.get("native_uri") or raw.get("uri"),
|
||||
retryable=_retryable_from_raw(BackendType.EVERMEMOS, raw),
|
||||
error_code=raw.get("error_code"),
|
||||
error_message=raw.get("error") or raw.get("error_message"),
|
||||
latency_ms=raw.get("latency_ms"),
|
||||
refs=refs,
|
||||
metadata=safe_backend_metadata(raw.get("metadata") or raw),
|
||||
)
|
||||
|
||||
|
||||
def normalize_openviking_ingest_response(raw: dict[str, Any]) -> BackendWriteResult:
|
||||
return _write_result(BackendType.OPENVIKING, raw)
|
||||
|
||||
|
||||
def normalize_evermemos_ingest_response(raw: dict[str, Any]) -> BackendWriteResult:
|
||||
return _write_result(BackendType.EVERMEMOS, raw)
|
||||
|
||||
|
||||
def normalize_openviking_retrieve_response(raw: dict[str, Any]) -> BackendRetrieveResult:
|
||||
return _retrieve_result(BackendType.OPENVIKING, raw)
|
||||
|
||||
|
||||
def normalize_evermemos_retrieve_response(raw: dict[str, Any]) -> BackendRetrieveResult:
|
||||
return _retrieve_result(BackendType.EVERMEMOS, raw)
|
||||
|
||||
|
||||
def map_backend_error_to_retryable(
|
||||
backend_type: BackendType,
|
||||
status_code: int | None = None,
|
||||
error_code: str | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> bool:
|
||||
"""Map backend errors into retryable/non-retryable categories.
|
||||
|
||||
Unknown errors default to retryable because adapter contracts are still
|
||||
unstable and transient backend/API rollout failures are more likely during
|
||||
integration.
|
||||
"""
|
||||
if status_code in {429, 500, 502, 503, 504}:
|
||||
return True
|
||||
if status_code in {400, 401, 403, 404, 422}:
|
||||
return False
|
||||
text = f"{error_code or ''} {error_message or ''}".lower()
|
||||
if "timeout" in text or "network_error" in text or "connection" in text:
|
||||
return True
|
||||
if "validation" in text or "unauthorized" in text or "forbidden" in text or "not_found" in text:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _write_result(backend_type: BackendType, raw: dict[str, Any]) -> BackendWriteResult:
|
||||
data = raw.get("result") or raw.get("data") or {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
native_id = (
|
||||
raw.get("native_id")
|
||||
or raw.get("id")
|
||||
or raw.get("memory_id")
|
||||
or raw.get("session_id")
|
||||
or data.get("native_id")
|
||||
or data.get("id")
|
||||
or data.get("memory_id")
|
||||
or data.get("session_id")
|
||||
)
|
||||
native_uri = (
|
||||
raw.get("native_uri")
|
||||
or raw.get("uri")
|
||||
or raw.get("url")
|
||||
or data.get("native_uri")
|
||||
or data.get("uri")
|
||||
or data.get("url")
|
||||
)
|
||||
if not native_uri and backend_type == BackendType.OPENVIKING and native_id:
|
||||
native_uri = f"viking://sessions/{native_id}"
|
||||
return BackendWriteResult(
|
||||
backend_type=backend_type,
|
||||
operation=BackendOperation.INGEST_TURN,
|
||||
status=_result_status(raw),
|
||||
native_id=native_id,
|
||||
native_uri=native_uri,
|
||||
retryable=_retryable_from_raw(backend_type, raw),
|
||||
error_code=raw.get("error_code"),
|
||||
error_message=raw.get("error") or raw.get("error_message"),
|
||||
latency_ms=raw.get("latency_ms"),
|
||||
metadata=safe_backend_metadata(raw.get("metadata") or raw),
|
||||
)
|
||||
|
||||
|
||||
def _retrieve_result(backend_type: BackendType, raw: dict[str, Any]) -> BackendRetrieveResult:
|
||||
if not isinstance(raw, dict) or not raw:
|
||||
return BackendRetrieveResult(
|
||||
backend_type=backend_type,
|
||||
operation=BackendOperation.RETRIEVE_CONTEXT,
|
||||
status=BackendResultStatus.SKIPPED,
|
||||
metadata={"reason": "malformed_or_empty_response"},
|
||||
)
|
||||
return BackendRetrieveResult(
|
||||
backend_type=backend_type,
|
||||
operation=BackendOperation.RETRIEVE_CONTEXT,
|
||||
status=_result_status(raw),
|
||||
native_id=raw.get("native_id") or raw.get("session_id"),
|
||||
native_uri=raw.get("native_uri") or raw.get("uri"),
|
||||
retryable=_retryable_from_raw(backend_type, raw),
|
||||
error_code=raw.get("error_code"),
|
||||
error_message=raw.get("error") or raw.get("error_message"),
|
||||
latency_ms=raw.get("latency_ms"),
|
||||
items=[_retrieve_item(backend_type, item) for item in _extract_retrieve_items(raw)],
|
||||
metadata=safe_backend_metadata(raw.get("metadata") or raw),
|
||||
)
|
||||
|
||||
|
||||
def _retrieve_item(backend_type: BackendType, item: dict[str, Any]) -> BackendRetrieveItem:
|
||||
metadata = safe_backend_metadata(item.get("metadata") if isinstance(item.get("metadata"), dict) else item)
|
||||
return BackendRetrieveItem(
|
||||
text=item.get("text") or item.get("summary") or item.get("abstract"),
|
||||
source_backend=backend_type,
|
||||
ref_id=item.get("ref_id") or item.get("id") or item.get("memory_id") or item.get("profile_id") or item.get("session_id") or item.get("uri"),
|
||||
score=float(item.get("score") or 0.0),
|
||||
memory_type=item.get("memory_type") or item.get("ref_type") or item.get("type") or item.get("kind"),
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
def _produced_ref(backend_type: BackendType, item: dict[str, Any]) -> BackendProducedRef:
|
||||
ref_type, mapping_metadata = map_backend_ref_type(backend_type, item.get("ref_type") or item.get("type") or item.get("kind"))
|
||||
metadata = {
|
||||
**safe_backend_metadata(item.get("metadata") if isinstance(item.get("metadata"), dict) else item),
|
||||
**mapping_metadata,
|
||||
}
|
||||
return BackendProducedRef(
|
||||
ref_type=ref_type,
|
||||
native_id=item.get("native_id") or item.get("id") or item.get("memory_id") or item.get("profile_id") or item.get("session_id"),
|
||||
native_uri=item.get("native_uri") or item.get("uri") or item.get("url"),
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
def _extract_ref_items(raw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
data = raw.get("result") or raw.get("data") or raw
|
||||
candidates = (
|
||||
data.get("refs")
|
||||
or data.get("produced_refs")
|
||||
or data.get("created_refs")
|
||||
or data.get("memories")
|
||||
or data.get("items")
|
||||
or []
|
||||
)
|
||||
return [item for item in candidates if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _extract_retrieve_items(raw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
data = raw.get("result") or raw.get("data") or raw
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
candidates = (
|
||||
data.get("items")
|
||||
or data.get("results")
|
||||
or data.get("memories")
|
||||
or data.get("resources")
|
||||
or data.get("contexts")
|
||||
or []
|
||||
)
|
||||
return [item for item in candidates if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _result_status(raw: dict[str, Any]) -> BackendResultStatus:
|
||||
status = str(raw.get("status") or "success").lower()
|
||||
if status in {"ok", "created", "accepted"}:
|
||||
return BackendResultStatus.SUCCESS
|
||||
try:
|
||||
return BackendResultStatus(status)
|
||||
except ValueError:
|
||||
return BackendResultStatus.SUCCESS if not raw.get("error") and not raw.get("error_message") else BackendResultStatus.FAILED
|
||||
|
||||
|
||||
def _retryable_from_raw(backend_type: BackendType, raw: dict[str, Any]) -> bool:
|
||||
if "retryable" in raw:
|
||||
return bool(raw["retryable"])
|
||||
if raw.get("error") or raw.get("error_message") or raw.get("error_code") or raw.get("status_code"):
|
||||
return map_backend_error_to_retryable(
|
||||
backend_type,
|
||||
status_code=raw.get("status_code"),
|
||||
error_code=raw.get("error_code"),
|
||||
error_message=raw.get("error") or raw.get("error_message"),
|
||||
)
|
||||
return False
|
||||
Reference in New Issue
Block a user