feat: extend memory search and attachment mapping
This commit is contained in:
61
README.md
61
README.md
@ -282,7 +282,8 @@ curl -X POST http://127.0.0.1:8010/resources \
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
对外返回的 `uri` 永远是 `resource://{user_id}/{resource_id}`,不会泄露内部 `file://` 路径。
|
资源上传接口返回的 `uri` 始终是 `resource://{user_id}/{resource_id}`。按文件名
|
||||||
|
命中的记忆搜索结果会另外通过 `attachments[].internal_uri` 返回真实 URI。
|
||||||
|
|
||||||
### 4. 查询资源列表
|
### 4. 查询资源列表
|
||||||
|
|
||||||
@ -438,10 +439,16 @@ Content-Type: application/json
|
|||||||
|---|---|---|---|---|
|
|---|---|---|---|---|
|
||||||
| `user_id` | string | 是 | 无 | 用户 ID |
|
| `user_id` | string | 是 | 无 | 用户 ID |
|
||||||
| `user_key` | string | 是 | 无 | 用户 key |
|
| `user_key` | string | 是 | 无 | 用户 key |
|
||||||
|
| `agent_id` | string | 否 | `null` | 设置后查询该 agent 的记忆;`user_id` 仍用于 Gateway 鉴权和本地数据隔离 |
|
||||||
| `query` | string | 是 | 无 | 搜索问题 |
|
| `query` | string | 是 | 无 | 搜索问题 |
|
||||||
| `conversation_id` | string | 否 | `null` | `scope` 包含 `current_chat` 时使用 |
|
| `conversation_id` | string | 否 | `null` | `scope` 包含 `current_chat` 时使用 |
|
||||||
| `scope` | string[] | 否 | `["current_chat", "resources"]` | 搜索范围 |
|
| `scope` | string[] | 否 | `["current_chat", "resources"]` | 搜索范围 |
|
||||||
| `top_k` | integer | 否 | `8` | 每次上游记忆服务搜索返回数量,范围 `1..100` |
|
| `method` | string | 否 | `hybrid` | 搜索方法:`keyword`、`vector`、`hybrid` 或 `agentic` |
|
||||||
|
| `top_k` | integer | 否 | `8` | 每次上游搜索返回数量,支持 `-1` 或 `1..100`;`-1` 表示使用上游默认数量 |
|
||||||
|
| `radius` | number | 否 | `null` | 相似度半径,范围 `0..1`;未提供时不发送给上游 |
|
||||||
|
| `include_profile` | boolean | 否 | `true` | 是否同时获取用户 profile;agent 查询由上游决定是否忽略该参数 |
|
||||||
|
| `enable_llm_rerank` | boolean | 否 | `true` | 是否启用上游 LLM rerank;具体生效范围由搜索方法和记忆类型决定 |
|
||||||
|
| `filters` | object | 否 | `null` | 上游过滤 DSL,支持字段条件以及嵌套 `AND`、`OR` |
|
||||||
| `app_id` | string | 否 | `default` | 上游记忆服务 app scope |
|
| `app_id` | string | 否 | `default` | 上游记忆服务 app scope |
|
||||||
| `project_id` | string | 否 | `default` | 上游记忆服务 project scope |
|
| `project_id` | string | 否 | `default` | 上游记忆服务 project scope |
|
||||||
|
|
||||||
@ -464,7 +471,13 @@ curl -X POST http://127.0.0.1:8010/memories/search \
|
|||||||
"conversation_id": "c_456",
|
"conversation_id": "c_456",
|
||||||
"query": "合同里的付款条款是什么?",
|
"query": "合同里的付款条款是什么?",
|
||||||
"scope": ["current_chat", "resources"],
|
"scope": ["current_chat", "resources"],
|
||||||
|
"method": "hybrid",
|
||||||
"top_k": 8,
|
"top_k": 8,
|
||||||
|
"include_profile": true,
|
||||||
|
"enable_llm_rerank": true,
|
||||||
|
"filters": {
|
||||||
|
"type": "Conversation"
|
||||||
|
},
|
||||||
"app_id": "default",
|
"app_id": "default",
|
||||||
"project_id": "default"
|
"project_id": "default"
|
||||||
}'
|
}'
|
||||||
@ -475,9 +488,39 @@ curl -X POST http://127.0.0.1:8010/memories/search \
|
|||||||
1. `current_chat`:调用上游记忆服务 search,过滤 `filters.session_id = chat:{conversation_id}`。
|
1. `current_chat`:调用上游记忆服务 search,过滤 `filters.session_id = chat:{conversation_id}`。
|
||||||
2. `resources`:先查当前用户的 `user_resources`,只取 `status = extracted` 且未删除资源;再按批次调用上游记忆服务 search,过滤这些资源的 `session_id`。
|
2. `resources`:先查当前用户的 `user_resources`,只取 `status = extracted` 且未删除资源;再按批次调用上游记忆服务 search,过滤这些资源的 `session_id`。
|
||||||
3. `all_user_memory`:调用上游记忆服务 search,不加 `session_id` 过滤。
|
3. `all_user_memory`:调用上游记忆服务 search,不加 `session_id` 过滤。
|
||||||
4. 合并结果。
|
4. 同时存在请求 `filters` 和 scope 生成的 session 条件时,使用 `{"AND": [filters, scope_filters]}` 合并,避免调用方过滤条件覆盖资源或聊天隔离条件。
|
||||||
5. 过滤 `memory_tombstones` 命中的 `memory_id` 或 `session_id`。
|
5. 设置 `agent_id` 时,上游请求只发送 `agent_id`;否则发送已鉴权的 `user_id`。
|
||||||
6. 应用 active `memory_overrides`,把 `text` 替换为 `override_text`。
|
6. 合并结果。
|
||||||
|
7. 过滤 `memory_tombstones` 命中的 `memory_id` 或 `session_id`。
|
||||||
|
8. 应用 active `memory_overrides`,把 `text` 替换为 `override_text`。
|
||||||
|
|
||||||
|
响应中的 `memory_type` 对应上游结果类型:
|
||||||
|
|
||||||
|
| `memory_type` | 说明 |
|
||||||
|
|---|---|
|
||||||
|
| `episode` | 用户 episode |
|
||||||
|
| `profile` | 用户 profile |
|
||||||
|
| `agent_case` | agent case |
|
||||||
|
| `agent_skill` | agent skill |
|
||||||
|
| `unprocessed_message` | 尚未完成边界提取的原始消息 |
|
||||||
|
|
||||||
|
附件路径映射规则:
|
||||||
|
|
||||||
|
1. `/resources` 上传成功后,将资源真实 URI 与资源 session 写入
|
||||||
|
`memory_attachments`。数据库初始化会自动回填已有 `user_resources`。
|
||||||
|
2. `/memories/add` 中含 `uri` 的 content item 会直接登记 URI。
|
||||||
|
3. `/memories/add` 中只有 `base64` 的 content item 会保存到
|
||||||
|
`MEMORY_GATEWAY_STORAGE_DIR/{user_id}/memory_attachments/{sha256}/`,再登记
|
||||||
|
生成的 `file://` URI。相同用户、session、文件名和内容的重试会复用路径。
|
||||||
|
4. 搜索时根据当前用户和结果 `session_id` 查询附件,递归检查 `raw` 中的字符串
|
||||||
|
值。只有完整文件名出现时才返回对应附件;匹配不区分大小写。
|
||||||
|
5. `raw` 中键名为 `base64` 的内容不会参与匹配。未匹配时返回
|
||||||
|
`"attachments": []`。
|
||||||
|
6. 历史 `/memories/add` 请求未保存在 Gateway 数据库中,无法自动补录映射;新
|
||||||
|
版本上线后的请求会建立映射。
|
||||||
|
|
||||||
|
`attachments[].internal_uri` 会按配置和调用方输入直接暴露服务器真实 URI,调用
|
||||||
|
该接口的客户端必须被视为可信客户端。
|
||||||
|
|
||||||
响应示例:
|
响应示例:
|
||||||
|
|
||||||
@ -486,12 +529,20 @@ curl -X POST http://127.0.0.1:8010/memories/search \
|
|||||||
"results": [
|
"results": [
|
||||||
{
|
{
|
||||||
"id": "mem_abc",
|
"id": "mem_abc",
|
||||||
|
"memory_type": "episode",
|
||||||
"session_id": "resource:u_123:r_xxx",
|
"session_id": "resource:u_123:r_xxx",
|
||||||
"text": "付款期限为收到发票后 30 天内。",
|
"text": "付款期限为收到发票后 30 天内。",
|
||||||
"score": 0.82,
|
"score": 0.82,
|
||||||
"source_scope": "resources",
|
"source_scope": "resources",
|
||||||
"resource_id": "r_xxx",
|
"resource_id": "r_xxx",
|
||||||
"resource_uri": "resource://u_123/r_xxx",
|
"resource_uri": "resource://u_123/r_xxx",
|
||||||
|
"attachments": [
|
||||||
|
{
|
||||||
|
"type": "pdf",
|
||||||
|
"name": "contract.pdf",
|
||||||
|
"internal_uri": "file:///srv/memory-gateway/u_123/r_xxx/contract.pdf"
|
||||||
|
}
|
||||||
|
],
|
||||||
"raw": {
|
"raw": {
|
||||||
"id": "mem_abc",
|
"id": "mem_abc",
|
||||||
"session_id": "resource:u_123:r_xxx",
|
"session_id": "resource:u_123:r_xxx",
|
||||||
|
|||||||
48
core/api.py
48
core/api.py
@ -8,14 +8,19 @@ from typing import Any, Literal
|
|||||||
from urllib.parse import parse_qsl, quote, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, quote, urlsplit, urlunsplit
|
||||||
|
|
||||||
from fastapi import APIRouter, FastAPI, File, Form, HTTPException, Request, UploadFile
|
from fastapi import APIRouter, FastAPI, File, Form, HTTPException, Request, UploadFile
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field, field_validator
|
||||||
from starlette.responses import Response
|
from starlette.responses import Response
|
||||||
|
|
||||||
from .config import GatewayConfig
|
from .config import GatewayConfig
|
||||||
from .db import init_db
|
from .db import init_db
|
||||||
from .backend_client import BackendClient
|
from .backend_client import BackendClient
|
||||||
from .repository import MemoryRepository
|
from .repository import MemoryRepository
|
||||||
from .service import MemoryGatewayService, UnsupportedContentType, UploadTooLarge
|
from .service import (
|
||||||
|
InvalidAttachment,
|
||||||
|
MemoryGatewayService,
|
||||||
|
UnsupportedContentType,
|
||||||
|
UploadTooLarge,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
API_LOGGER = logging.getLogger("memory_gateway.api")
|
API_LOGGER = logging.getLogger("memory_gateway.api")
|
||||||
@ -34,15 +39,28 @@ SENSITIVE_FIELD_NAMES = {
|
|||||||
class SearchMemoriesRequest(BaseModel):
|
class SearchMemoriesRequest(BaseModel):
|
||||||
user_id: str = Field(min_length=1)
|
user_id: str = Field(min_length=1)
|
||||||
user_key: str = Field(min_length=1)
|
user_key: str = Field(min_length=1)
|
||||||
|
agent_id: str | None = Field(default=None, min_length=1)
|
||||||
conversation_id: str | None = None
|
conversation_id: str | None = None
|
||||||
query: str = Field(min_length=1)
|
query: str = Field(min_length=1)
|
||||||
scope: list[Literal["current_chat", "resources", "all_user_memory"]] = Field(
|
scope: list[Literal["current_chat", "resources", "all_user_memory"]] = Field(
|
||||||
default_factory=lambda: ["current_chat", "resources"]
|
default_factory=lambda: ["current_chat", "resources"]
|
||||||
)
|
)
|
||||||
top_k: int = Field(default=8, ge=1, le=100)
|
method: Literal["keyword", "vector", "hybrid", "agentic"] = "hybrid"
|
||||||
|
top_k: int = 8
|
||||||
|
radius: float | None = Field(default=None, ge=0, le=1)
|
||||||
|
include_profile: bool = True
|
||||||
|
enable_llm_rerank: bool = True
|
||||||
|
filters: dict[str, Any] | None = None
|
||||||
app_id: str = "default"
|
app_id: str = "default"
|
||||||
project_id: str = "default"
|
project_id: str = "default"
|
||||||
|
|
||||||
|
@field_validator("top_k")
|
||||||
|
@classmethod
|
||||||
|
def validate_top_k(cls, value: int) -> int:
|
||||||
|
if value != -1 and not 1 <= value <= 100:
|
||||||
|
raise ValueError("top_k must be -1 or in 1..100")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
class AddMemoryMessage(BaseModel):
|
class AddMemoryMessage(BaseModel):
|
||||||
sender_id: str = Field(min_length=1)
|
sender_id: str = Field(min_length=1)
|
||||||
@ -367,10 +385,16 @@ def create_app(
|
|||||||
require_user(request.user_id, request.user_key)
|
require_user(request.user_id, request.user_key)
|
||||||
return await service.search_memories(
|
return await service.search_memories(
|
||||||
user_id=request.user_id,
|
user_id=request.user_id,
|
||||||
|
agent_id=request.agent_id,
|
||||||
query=request.query,
|
query=request.query,
|
||||||
conversation_id=request.conversation_id,
|
conversation_id=request.conversation_id,
|
||||||
scope=request.scope,
|
scope=request.scope,
|
||||||
|
method=request.method,
|
||||||
top_k=request.top_k,
|
top_k=request.top_k,
|
||||||
|
radius=request.radius,
|
||||||
|
include_profile=request.include_profile,
|
||||||
|
enable_llm_rerank=request.enable_llm_rerank,
|
||||||
|
filters=request.filters,
|
||||||
app_id=request.app_id,
|
app_id=request.app_id,
|
||||||
project_id=request.project_id,
|
project_id=request.project_id,
|
||||||
)
|
)
|
||||||
@ -380,12 +404,18 @@ def create_app(
|
|||||||
request: AddMemoryRequest,
|
request: AddMemoryRequest,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
require_user(request.user_id, request.user_key)
|
require_user(request.user_id, request.user_key)
|
||||||
return await service.add_memory(
|
try:
|
||||||
session_id=request.session_id,
|
return await service.add_memory(
|
||||||
app_id=request.app_id,
|
user_id=request.user_id,
|
||||||
project_id=request.project_id,
|
session_id=request.session_id,
|
||||||
messages=[message.model_dump() for message in request.messages],
|
app_id=request.app_id,
|
||||||
)
|
project_id=request.project_id,
|
||||||
|
messages=[message.model_dump() for message in request.messages],
|
||||||
|
)
|
||||||
|
except UploadTooLarge as exc:
|
||||||
|
raise HTTPException(status_code=413, detail=str(exc)) from exc
|
||||||
|
except InvalidAttachment as exc:
|
||||||
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||||
|
|
||||||
@router.post("/memories/flush")
|
@router.post("/memories/flush")
|
||||||
async def flush_memory(
|
async def flush_memory(
|
||||||
|
|||||||
56
core/db.py
56
core/db.py
@ -43,6 +43,62 @@ ON user_resources (session_id);
|
|||||||
CREATE INDEX IF NOT EXISTS idx_user_resources_user_id
|
CREATE INDEX IF NOT EXISTS idx_user_resources_user_id
|
||||||
ON user_resources (user_id);
|
ON user_resources (user_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS memory_attachments (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
app_id TEXT NOT NULL DEFAULT 'default',
|
||||||
|
project_id TEXT NOT NULL DEFAULT 'default',
|
||||||
|
session_id TEXT NOT NULL,
|
||||||
|
resource_id TEXT,
|
||||||
|
content_type TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
internal_uri TEXT NOT NULL,
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
sha256 TEXT,
|
||||||
|
created_at TIMESTAMP NOT NULL,
|
||||||
|
deleted_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_attachments_unique_uri
|
||||||
|
ON memory_attachments (user_id, session_id, internal_uri);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_memory_attachments_user_session
|
||||||
|
ON memory_attachments (user_id, session_id, deleted_at);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_memory_attachments_resource
|
||||||
|
ON memory_attachments (resource_id, deleted_at);
|
||||||
|
|
||||||
|
INSERT OR IGNORE INTO memory_attachments (
|
||||||
|
id,
|
||||||
|
user_id,
|
||||||
|
app_id,
|
||||||
|
project_id,
|
||||||
|
session_id,
|
||||||
|
resource_id,
|
||||||
|
content_type,
|
||||||
|
name,
|
||||||
|
internal_uri,
|
||||||
|
source,
|
||||||
|
sha256,
|
||||||
|
created_at,
|
||||||
|
deleted_at
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
'a_resource_' || id,
|
||||||
|
user_id,
|
||||||
|
app_id,
|
||||||
|
project_id,
|
||||||
|
session_id,
|
||||||
|
id,
|
||||||
|
content_type,
|
||||||
|
COALESCE(original_filename, id),
|
||||||
|
uri,
|
||||||
|
'resource_upload',
|
||||||
|
sha256,
|
||||||
|
created_at,
|
||||||
|
deleted_at
|
||||||
|
FROM user_resources;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS memory_tombstones (
|
CREATE TABLE IF NOT EXISTS memory_tombstones (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
user_id TEXT NOT NULL,
|
user_id TEXT NOT NULL,
|
||||||
|
|||||||
@ -96,9 +96,13 @@ class MemoryRepository:
|
|||||||
now = utc_now()
|
now = utc_now()
|
||||||
where = "id = ? AND deleted_at IS NULL"
|
where = "id = ? AND deleted_at IS NULL"
|
||||||
params: tuple[Any, ...] = (now, now, resource_id)
|
params: tuple[Any, ...] = (now, now, resource_id)
|
||||||
|
attachment_where = "resource_id = ? AND deleted_at IS NULL"
|
||||||
|
attachment_params: tuple[Any, ...] = (now, resource_id)
|
||||||
if user_id is not None:
|
if user_id is not None:
|
||||||
where += " AND user_id = ?"
|
where += " AND user_id = ?"
|
||||||
params = (now, now, resource_id, user_id)
|
params = (now, now, resource_id, user_id)
|
||||||
|
attachment_where += " AND user_id = ?"
|
||||||
|
attachment_params = (now, resource_id, user_id)
|
||||||
with connect(self.db_path) as conn:
|
with connect(self.db_path) as conn:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
f"""
|
f"""
|
||||||
@ -108,6 +112,14 @@ class MemoryRepository:
|
|||||||
""",
|
""",
|
||||||
params,
|
params,
|
||||||
)
|
)
|
||||||
|
conn.execute(
|
||||||
|
f"""
|
||||||
|
UPDATE memory_attachments
|
||||||
|
SET deleted_at = ?
|
||||||
|
WHERE {attachment_where}
|
||||||
|
""",
|
||||||
|
attachment_params,
|
||||||
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
return self.get_resource(resource_id)
|
return self.get_resource(resource_id)
|
||||||
|
|
||||||
@ -215,6 +227,62 @@ class MemoryRepository:
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
return [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
def create_attachment(self, **values: Any) -> dict[str, Any]:
|
||||||
|
attachment_id = str(values.get("id") or f"a_{uuid.uuid4().hex}")
|
||||||
|
payload = {
|
||||||
|
"id": attachment_id,
|
||||||
|
"created_at": utc_now(),
|
||||||
|
"deleted_at": None,
|
||||||
|
**values,
|
||||||
|
}
|
||||||
|
with connect(self.db_path) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT OR IGNORE INTO memory_attachments (
|
||||||
|
id, user_id, app_id, project_id, session_id, resource_id,
|
||||||
|
content_type, name, internal_uri, source, sha256,
|
||||||
|
created_at, deleted_at
|
||||||
|
) VALUES (
|
||||||
|
:id, :user_id, :app_id, :project_id, :session_id, :resource_id,
|
||||||
|
:content_type, :name, :internal_uri, :source, :sha256,
|
||||||
|
:created_at, :deleted_at
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
payload,
|
||||||
|
)
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM memory_attachments
|
||||||
|
WHERE user_id = ? AND session_id = ? AND internal_uri = ?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
payload["user_id"],
|
||||||
|
payload["session_id"],
|
||||||
|
payload["internal_uri"],
|
||||||
|
),
|
||||||
|
).fetchone()
|
||||||
|
conn.commit()
|
||||||
|
attachment = _row_to_dict(row)
|
||||||
|
if attachment is None:
|
||||||
|
raise RuntimeError("created attachment could not be read back")
|
||||||
|
return attachment
|
||||||
|
|
||||||
|
def list_attachments_for_session(
|
||||||
|
self,
|
||||||
|
user_id: str,
|
||||||
|
session_id: str,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
with connect(self.db_path) as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM memory_attachments
|
||||||
|
WHERE user_id = ? AND session_id = ? AND deleted_at IS NULL
|
||||||
|
ORDER BY created_at ASC, id ASC
|
||||||
|
""",
|
||||||
|
(user_id, session_id),
|
||||||
|
).fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
def add_tombstone(
|
def add_tombstone(
|
||||||
self,
|
self,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
|
|||||||
268
core/service.py
268
core/service.py
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
|
import binascii
|
||||||
import hashlib
|
import hashlib
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import secrets
|
import secrets
|
||||||
@ -63,6 +64,10 @@ class UnsupportedContentType(ValueError):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidAttachment(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _copy_upload(
|
def _copy_upload(
|
||||||
file: UploadFile,
|
file: UploadFile,
|
||||||
destination: Path,
|
destination: Path,
|
||||||
@ -180,6 +185,7 @@ class MemoryGatewayService:
|
|||||||
)
|
)
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
shutil.rmtree(stored_path.parent, ignore_errors=True)
|
shutil.rmtree(stored_path.parent, ignore_errors=True)
|
||||||
|
self._register_resource_attachment(existing)
|
||||||
return self._resource_summary(existing)
|
return self._resource_summary(existing)
|
||||||
|
|
||||||
internal_uri = stored_path.resolve().as_uri()
|
internal_uri = stored_path.resolve().as_uri()
|
||||||
@ -202,6 +208,7 @@ class MemoryGatewayService:
|
|||||||
status="ingesting",
|
status="ingesting",
|
||||||
error_message=None,
|
error_message=None,
|
||||||
)
|
)
|
||||||
|
self._register_resource_attachment(resource)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await self._retry_backend_call(
|
await self._retry_backend_call(
|
||||||
@ -346,10 +353,16 @@ class MemoryGatewayService:
|
|||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
|
agent_id: str | None,
|
||||||
query: str,
|
query: str,
|
||||||
conversation_id: str | None,
|
conversation_id: str | None,
|
||||||
scope: list[str],
|
scope: list[str],
|
||||||
|
method: str,
|
||||||
top_k: int,
|
top_k: int,
|
||||||
|
radius: float | None,
|
||||||
|
include_profile: bool,
|
||||||
|
enable_llm_rerank: bool,
|
||||||
|
filters: dict[str, Any] | None,
|
||||||
app_id: str,
|
app_id: str,
|
||||||
project_id: str,
|
project_id: str,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
@ -359,11 +372,19 @@ class MemoryGatewayService:
|
|||||||
if "current_chat" in scope and conversation_id:
|
if "current_chat" in scope and conversation_id:
|
||||||
payload = self._search_payload(
|
payload = self._search_payload(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
|
agent_id=agent_id,
|
||||||
query=query,
|
query=query,
|
||||||
|
method=method,
|
||||||
top_k=top_k,
|
top_k=top_k,
|
||||||
|
radius=radius,
|
||||||
|
include_profile=include_profile,
|
||||||
|
enable_llm_rerank=enable_llm_rerank,
|
||||||
app_id=app_id,
|
app_id=app_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
filters={"session_id": f"chat:{conversation_id}"},
|
filters=_combine_filters(
|
||||||
|
filters,
|
||||||
|
{"session_id": f"chat:{conversation_id}"},
|
||||||
|
),
|
||||||
)
|
)
|
||||||
results.extend(
|
results.extend(
|
||||||
self._extract_results(
|
self._extract_results(
|
||||||
@ -385,11 +406,19 @@ class MemoryGatewayService:
|
|||||||
for batch in _chunks(session_ids, self.config.resource_search_batch_size):
|
for batch in _chunks(session_ids, self.config.resource_search_batch_size):
|
||||||
payload = self._search_payload(
|
payload = self._search_payload(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
|
agent_id=agent_id,
|
||||||
query=query,
|
query=query,
|
||||||
|
method=method,
|
||||||
top_k=top_k,
|
top_k=top_k,
|
||||||
|
radius=radius,
|
||||||
|
include_profile=include_profile,
|
||||||
|
enable_llm_rerank=enable_llm_rerank,
|
||||||
app_id=app_id,
|
app_id=app_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
filters={"session_id": {"in": batch}},
|
filters=_combine_filters(
|
||||||
|
filters,
|
||||||
|
{"session_id": {"in": batch}},
|
||||||
|
),
|
||||||
)
|
)
|
||||||
results.extend(
|
results.extend(
|
||||||
self._extract_results(
|
self._extract_results(
|
||||||
@ -403,11 +432,16 @@ class MemoryGatewayService:
|
|||||||
if "all_user_memory" in scope:
|
if "all_user_memory" in scope:
|
||||||
payload = self._search_payload(
|
payload = self._search_payload(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
|
agent_id=agent_id,
|
||||||
query=query,
|
query=query,
|
||||||
|
method=method,
|
||||||
top_k=top_k,
|
top_k=top_k,
|
||||||
|
radius=radius,
|
||||||
|
include_profile=include_profile,
|
||||||
|
enable_llm_rerank=enable_llm_rerank,
|
||||||
app_id=app_id,
|
app_id=app_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
filters=None,
|
filters=filters,
|
||||||
)
|
)
|
||||||
results.extend(
|
results.extend(
|
||||||
self._extract_results(
|
self._extract_results(
|
||||||
@ -425,21 +459,126 @@ class MemoryGatewayService:
|
|||||||
async def add_memory(
|
async def add_memory(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
|
user_id: str,
|
||||||
session_id: str,
|
session_id: str,
|
||||||
app_id: str,
|
app_id: str,
|
||||||
project_id: str,
|
project_id: str,
|
||||||
messages: list[dict[str, Any]],
|
messages: list[dict[str, Any]],
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
|
attachments, generated_paths = self._prepare_memory_attachments(
|
||||||
|
user_id=user_id,
|
||||||
|
session_id=session_id,
|
||||||
|
app_id=app_id,
|
||||||
|
project_id=project_id,
|
||||||
|
messages=messages,
|
||||||
|
)
|
||||||
payload = {
|
payload = {
|
||||||
"session_id": session_id,
|
"session_id": session_id,
|
||||||
"app_id": app_id,
|
"app_id": app_id,
|
||||||
"project_id": project_id,
|
"project_id": project_id,
|
||||||
"messages": messages,
|
"messages": messages,
|
||||||
}
|
}
|
||||||
return {
|
try:
|
||||||
"session_id": session_id,
|
backend = await self.backend_client.add_memory(payload)
|
||||||
"backend": await self.backend_client.add_memory(payload),
|
for attachment in attachments:
|
||||||
}
|
self.repository.create_attachment(**attachment)
|
||||||
|
except Exception:
|
||||||
|
for path in generated_paths:
|
||||||
|
path.unlink(missing_ok=True)
|
||||||
|
_remove_empty_parents(path.parent, stop_at=self.config.storage_dir)
|
||||||
|
raise
|
||||||
|
return {"session_id": session_id, "backend": backend}
|
||||||
|
|
||||||
|
def _register_resource_attachment(self, resource: dict[str, Any]) -> None:
|
||||||
|
self.repository.create_attachment(
|
||||||
|
user_id=resource["user_id"],
|
||||||
|
app_id=resource["app_id"],
|
||||||
|
project_id=resource["project_id"],
|
||||||
|
session_id=resource["session_id"],
|
||||||
|
resource_id=resource["id"],
|
||||||
|
content_type=resource["content_type"],
|
||||||
|
name=resource["original_filename"] or resource["id"],
|
||||||
|
internal_uri=resource["uri"],
|
||||||
|
source="resource_upload",
|
||||||
|
sha256=resource["sha256"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def _prepare_memory_attachments(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
user_id: str,
|
||||||
|
session_id: str,
|
||||||
|
app_id: str,
|
||||||
|
project_id: str,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
) -> tuple[list[dict[str, Any]], list[Path]]:
|
||||||
|
attachments: list[dict[str, Any]] = []
|
||||||
|
generated_paths: list[Path] = []
|
||||||
|
try:
|
||||||
|
for message in messages:
|
||||||
|
content = message.get("content")
|
||||||
|
if not isinstance(content, list):
|
||||||
|
continue
|
||||||
|
for item in content:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
uri = item.get("uri")
|
||||||
|
encoded = item.get("base64")
|
||||||
|
if not uri and not encoded:
|
||||||
|
continue
|
||||||
|
attachment_id = f"a_{uuid.uuid4().hex}"
|
||||||
|
name = _attachment_name(item, str(uri) if uri else None)
|
||||||
|
sha256: str | None = None
|
||||||
|
if uri:
|
||||||
|
internal_uri = str(uri)
|
||||||
|
source = "memory_add_uri"
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
data = base64.b64decode(str(encoded), validate=True)
|
||||||
|
except (binascii.Error, ValueError) as exc:
|
||||||
|
raise InvalidAttachment(
|
||||||
|
f"invalid base64 attachment: {name}"
|
||||||
|
) from exc
|
||||||
|
if len(data) > self.config.max_upload_bytes:
|
||||||
|
raise UploadTooLarge(
|
||||||
|
f"attachment exceeds max size of "
|
||||||
|
f"{self.config.max_upload_bytes} bytes"
|
||||||
|
)
|
||||||
|
sha256 = hashlib.sha256(data).hexdigest()
|
||||||
|
path = (
|
||||||
|
self.config.storage_dir
|
||||||
|
/ user_id
|
||||||
|
/ "memory_attachments"
|
||||||
|
/ sha256
|
||||||
|
/ name
|
||||||
|
)
|
||||||
|
if not path.exists():
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_bytes(data)
|
||||||
|
generated_paths.append(path)
|
||||||
|
internal_uri = path.resolve().as_uri()
|
||||||
|
source = "memory_add_base64"
|
||||||
|
attachments.append(
|
||||||
|
{
|
||||||
|
"id": attachment_id,
|
||||||
|
"user_id": user_id,
|
||||||
|
"app_id": app_id,
|
||||||
|
"project_id": project_id,
|
||||||
|
"session_id": session_id,
|
||||||
|
"resource_id": None,
|
||||||
|
"content_type": str(item.get("type") or "doc"),
|
||||||
|
"name": name,
|
||||||
|
"internal_uri": internal_uri,
|
||||||
|
"source": source,
|
||||||
|
"sha256": sha256,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
for path in generated_paths:
|
||||||
|
path.unlink(missing_ok=True)
|
||||||
|
_remove_empty_parents(path.parent, stop_at=self.config.storage_dir)
|
||||||
|
raise
|
||||||
|
return attachments, generated_paths
|
||||||
|
|
||||||
async def flush_memory(
|
async def flush_memory(
|
||||||
self,
|
self,
|
||||||
@ -461,19 +600,29 @@ class MemoryGatewayService:
|
|||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
user_id: str,
|
user_id: str,
|
||||||
|
agent_id: str | None,
|
||||||
query: str,
|
query: str,
|
||||||
|
method: str,
|
||||||
top_k: int,
|
top_k: int,
|
||||||
|
radius: float | None,
|
||||||
|
include_profile: bool,
|
||||||
|
enable_llm_rerank: bool,
|
||||||
app_id: str,
|
app_id: str,
|
||||||
project_id: str,
|
project_id: str,
|
||||||
filters: dict[str, Any] | None,
|
filters: dict[str, Any] | None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
payload: dict[str, Any] = {
|
payload: dict[str, Any] = {
|
||||||
"user_id": user_id,
|
|
||||||
"query": query,
|
"query": query,
|
||||||
|
"method": method,
|
||||||
"top_k": top_k,
|
"top_k": top_k,
|
||||||
|
"include_profile": include_profile,
|
||||||
|
"enable_llm_rerank": enable_llm_rerank,
|
||||||
"app_id": app_id,
|
"app_id": app_id,
|
||||||
"project_id": project_id,
|
"project_id": project_id,
|
||||||
}
|
}
|
||||||
|
payload["agent_id" if agent_id else "user_id"] = agent_id or user_id
|
||||||
|
if radius is not None:
|
||||||
|
payload["radius"] = radius
|
||||||
if filters is not None:
|
if filters is not None:
|
||||||
payload["filters"] = filters
|
payload["filters"] = filters
|
||||||
return payload
|
return payload
|
||||||
@ -487,18 +636,22 @@ class MemoryGatewayService:
|
|||||||
user_id: str,
|
user_id: str,
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
data = response.get("data", {})
|
data = response.get("data", {})
|
||||||
raw_items: list[dict[str, Any]] = []
|
raw_items: list[tuple[str, dict[str, Any]]] = []
|
||||||
for key in (
|
memory_types = {
|
||||||
"episodes",
|
"episodes": "episode",
|
||||||
"profiles",
|
"profiles": "profile",
|
||||||
"agent_cases",
|
"agent_cases": "agent_case",
|
||||||
"agent_skills",
|
"agent_skills": "agent_skill",
|
||||||
"unprocessed_messages",
|
"unprocessed_messages": "unprocessed_message",
|
||||||
):
|
}
|
||||||
raw_items.extend(data.get(key, []) or [])
|
for key, memory_type in memory_types.items():
|
||||||
|
raw_items.extend(
|
||||||
|
(memory_type, item) for item in (data.get(key, []) or [])
|
||||||
|
)
|
||||||
|
|
||||||
normalized = []
|
normalized = []
|
||||||
for raw in raw_items:
|
attachment_cache: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for memory_type, raw in raw_items:
|
||||||
session_id = raw.get("session_id")
|
session_id = raw.get("session_id")
|
||||||
resource = session_resource_map.get(session_id)
|
resource = session_resource_map.get(session_id)
|
||||||
if resource is None and isinstance(session_id, str):
|
if resource is None and isinstance(session_id, str):
|
||||||
@ -506,9 +659,21 @@ class MemoryGatewayService:
|
|||||||
session_id,
|
session_id,
|
||||||
user_id,
|
user_id,
|
||||||
)
|
)
|
||||||
|
attachments: list[dict[str, Any]] = []
|
||||||
|
if isinstance(session_id, str):
|
||||||
|
if session_id not in attachment_cache:
|
||||||
|
attachment_cache[session_id] = (
|
||||||
|
self.repository.list_attachments_for_session(
|
||||||
|
user_id,
|
||||||
|
session_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session_attachments = attachment_cache[session_id]
|
||||||
|
attachments = _matching_attachments(raw, session_attachments)
|
||||||
normalized.append(
|
normalized.append(
|
||||||
{
|
{
|
||||||
"id": raw.get("id"),
|
"id": raw.get("id"),
|
||||||
|
"memory_type": memory_type,
|
||||||
"session_id": session_id,
|
"session_id": session_id,
|
||||||
"text": _display_text(raw),
|
"text": _display_text(raw),
|
||||||
"score": raw.get("score"),
|
"score": raw.get("score"),
|
||||||
@ -517,6 +682,7 @@ class MemoryGatewayService:
|
|||||||
"resource_uri": (
|
"resource_uri": (
|
||||||
public_resource_uri(user_id, resource["id"]) if resource else None
|
public_resource_uri(user_id, resource["id"]) if resource else None
|
||||||
),
|
),
|
||||||
|
"attachments": attachments,
|
||||||
"raw": raw,
|
"raw": raw,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@ -623,6 +789,72 @@ class MemoryGatewayService:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _combine_filters(
|
||||||
|
custom_filters: dict[str, Any] | None,
|
||||||
|
scope_filters: dict[str, Any] | None,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
if custom_filters is None:
|
||||||
|
return scope_filters
|
||||||
|
if scope_filters is None:
|
||||||
|
return custom_filters
|
||||||
|
return {"AND": [custom_filters, scope_filters]}
|
||||||
|
|
||||||
|
|
||||||
|
def _attachment_name(item: dict[str, Any], uri: str | None) -> str:
|
||||||
|
if item.get("name"):
|
||||||
|
return _safe_filename(str(item["name"]))
|
||||||
|
if uri:
|
||||||
|
parsed = urlparse(uri)
|
||||||
|
uri_name = Path(unquote(parsed.path)).name
|
||||||
|
if uri_name:
|
||||||
|
return _safe_filename(uri_name)
|
||||||
|
extension = str(item.get("ext") or "bin").lstrip(".") or "bin"
|
||||||
|
return f"attachment.{extension}"
|
||||||
|
|
||||||
|
|
||||||
|
def _matching_attachments(
|
||||||
|
raw: dict[str, Any],
|
||||||
|
attachments: list[dict[str, Any]],
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
strings = [value.casefold() for value in _raw_string_values(raw)]
|
||||||
|
matched: list[dict[str, Any]] = []
|
||||||
|
seen_uris: set[str] = set()
|
||||||
|
for attachment in attachments:
|
||||||
|
name = str(attachment["name"])
|
||||||
|
internal_uri = str(attachment["internal_uri"])
|
||||||
|
if internal_uri in seen_uris:
|
||||||
|
continue
|
||||||
|
if not any(name.casefold() in value for value in strings):
|
||||||
|
continue
|
||||||
|
seen_uris.add(internal_uri)
|
||||||
|
matched.append(
|
||||||
|
{
|
||||||
|
"type": attachment["content_type"],
|
||||||
|
"name": name,
|
||||||
|
"internal_uri": internal_uri,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return matched
|
||||||
|
|
||||||
|
|
||||||
|
def _raw_string_values(value: Any, key: str | None = None) -> list[str]:
|
||||||
|
if key is not None and key.casefold() == "base64":
|
||||||
|
return []
|
||||||
|
if isinstance(value, str):
|
||||||
|
return [value]
|
||||||
|
if isinstance(value, dict):
|
||||||
|
strings: list[str] = []
|
||||||
|
for item_key, item_value in value.items():
|
||||||
|
strings.extend(_raw_string_values(item_value, str(item_key)))
|
||||||
|
return strings
|
||||||
|
if isinstance(value, list):
|
||||||
|
strings = []
|
||||||
|
for item in value:
|
||||||
|
strings.extend(_raw_string_values(item))
|
||||||
|
return strings
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
def _chunks(items: list[str], size: int) -> list[list[str]]:
|
def _chunks(items: list[str], size: int) -> list[list[str]]:
|
||||||
if not items:
|
if not items:
|
||||||
return []
|
return []
|
||||||
|
|||||||
@ -0,0 +1,57 @@
|
|||||||
|
# Memory Attachment Path Mapping Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Persist attachment-to-session mappings for resource and direct memory ingestion, then return filename-matched real URIs from memory search results.
|
||||||
|
|
||||||
|
**Architecture:** Add one SQLite attachment table and repository methods. Register resource files directly, materialize base64 memory attachments under Gateway storage, and enrich normalized search results by matching attachment names against recursive raw string values.
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.10+, FastAPI, SQLite, Pydantic, pytest, httpx.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Attachment persistence
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/db.py`
|
||||||
|
- Modify: `core/repository.py`
|
||||||
|
- Modify: `tests/test_gateway.py`
|
||||||
|
|
||||||
|
- [x] Write failing tests proving attachment records can be created, listed by user/session, deduplicated, and soft-deleted with resources.
|
||||||
|
- [x] Run focused tests and verify failure because the table and methods do not exist.
|
||||||
|
- [x] Add `memory_attachments`, indexes, resource backfill SQL, and focused repository methods.
|
||||||
|
- [x] Run focused tests and verify they pass.
|
||||||
|
|
||||||
|
### Task 2: Register attachments during ingestion
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/api.py`
|
||||||
|
- Modify: `core/service.py`
|
||||||
|
- Modify: `tests/test_gateway.py`
|
||||||
|
|
||||||
|
- [x] Write failing tests for `/resources`, `/memories/add` URI items, and `/memories/add` base64 items.
|
||||||
|
- [x] Run focused tests and verify missing mappings and files.
|
||||||
|
- [x] Register resource mappings, pass authenticated `user_id` into add service, materialize base64 files, and persist successful add mappings.
|
||||||
|
- [x] Run focused tests and verify they pass.
|
||||||
|
|
||||||
|
### Task 3: Enrich search results
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/service.py`
|
||||||
|
- Modify: `tests/test_gateway.py`
|
||||||
|
|
||||||
|
- [x] Write failing tests for filename match, no match, base64-key exclusion, and cross-user isolation.
|
||||||
|
- [x] Run focused tests and verify `attachments` is absent.
|
||||||
|
- [x] Recursively collect raw strings excluding base64 and return deduplicated matching attachments.
|
||||||
|
- [x] Run focused tests and verify they pass.
|
||||||
|
|
||||||
|
### Task 4: Documentation and regression
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `README.md`
|
||||||
|
- Modify: `tests/test_command.md`
|
||||||
|
|
||||||
|
- [x] Document attachment persistence, historical backfill limits, matching behavior, and response shape.
|
||||||
|
- [x] Update the search response example with `attachments`.
|
||||||
|
- [x] Run `git diff --check`, compile checks, and the complete pytest suite.
|
||||||
|
- [x] Review the final diff for user isolation and unintended URI exposure outside search.
|
||||||
@ -0,0 +1,118 @@
|
|||||||
|
# Memory Search Upstream Options Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Extend `POST /memories/search` with all upstream search options while preserving Gateway authentication, scopes, resource isolation, tombstones, and overrides.
|
||||||
|
|
||||||
|
**Architecture:** Extend the existing Pydantic request model and pass the validated values through `MemoryGatewayService`. Keep scope orchestration intact, combine caller filters with scope-generated session filters using `AND`, and tag normalized results according to their upstream response array.
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.10+, FastAPI, Pydantic v2, pytest, pytest-asyncio, httpx ASGI transport.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Search request options and defaults
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `tests/test_gateway.py`
|
||||||
|
- Modify: `core/api.py`
|
||||||
|
- Modify: `core/service.py`
|
||||||
|
|
||||||
|
- [x] **Step 1: Write failing tests for defaults, custom options, and validation**
|
||||||
|
|
||||||
|
Add API tests that assert a default search sends `method="hybrid"`, `include_profile=true`, and `enable_llm_rerank=true`; a custom request forwards `agent_id`, `keyword`, `radius`, `top_k=-1`, and both false flags; and invalid `method`, `radius`, and `top_k=0` return HTTP 422.
|
||||||
|
|
||||||
|
- [x] **Step 2: Run tests and verify expected failures**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run pytest tests/test_gateway.py -k 'search_forwards_default_upstream_options or search_forwards_all_upstream_options or search_rejects_invalid_upstream_options' -q
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: assertions fail because the request model and service do not yet accept or forward the new fields.
|
||||||
|
|
||||||
|
- [x] **Step 3: Implement request fields and payload forwarding**
|
||||||
|
|
||||||
|
Extend `SearchMemoriesRequest` with:
|
||||||
|
|
||||||
|
```python
|
||||||
|
agent_id: str | None = Field(default=None, min_length=1)
|
||||||
|
method: Literal["keyword", "vector", "hybrid", "agentic"] = "hybrid"
|
||||||
|
radius: float | None = Field(default=None, ge=0, le=1)
|
||||||
|
include_profile: bool = True
|
||||||
|
enable_llm_rerank: bool = True
|
||||||
|
filters: dict[str, Any] | None = None
|
||||||
|
```
|
||||||
|
|
||||||
|
Validate `top_k` as `-1` or `1..100`, pass all values to the service, and make `_search_payload` select exactly one upstream owner key (`agent_id` when present, otherwise `user_id`).
|
||||||
|
|
||||||
|
- [x] **Step 4: Run focused tests and verify they pass**
|
||||||
|
|
||||||
|
Run the command from Step 2. Expected: all selected tests pass.
|
||||||
|
|
||||||
|
### Task 2: Filter composition and result memory types
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `tests/test_gateway.py`
|
||||||
|
- Modify: `core/service.py`
|
||||||
|
|
||||||
|
- [x] **Step 1: Write failing tests for filter composition and result types**
|
||||||
|
|
||||||
|
Add a resource-scope test asserting caller filters and `session_id in [...]` are combined as:
|
||||||
|
|
||||||
|
```python
|
||||||
|
{"AND": [caller_filters, {"session_id": {"in": [session_id]}}]}
|
||||||
|
```
|
||||||
|
|
||||||
|
Extend the fake backend to return all response arrays and assert normalized results have `memory_type` values `episode`, `profile`, `agent_case`, `agent_skill`, and `unprocessed_message`.
|
||||||
|
|
||||||
|
- [x] **Step 2: Run tests and verify expected failures**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run pytest tests/test_gateway.py -k 'search_combines_custom_and_scope_filters or search_labels_all_memory_types' -q
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: failures because caller filters are not composed and normalized results have no `memory_type`.
|
||||||
|
|
||||||
|
- [x] **Step 3: Implement composition and typed normalization**
|
||||||
|
|
||||||
|
Add a small `_combine_filters` helper that returns either condition directly, returns `None` when both are absent, or returns `{"AND": [custom, scope]}` when both exist. Iterate an explicit mapping from response array name to memory type in `_extract_results` and include the mapped value in every normalized result.
|
||||||
|
|
||||||
|
- [x] **Step 4: Run focused tests and verify they pass**
|
||||||
|
|
||||||
|
Run the command from Step 2. Expected: both tests pass.
|
||||||
|
|
||||||
|
### Task 3: Documentation and regression verification
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `README.md`
|
||||||
|
- Verify: `tests/test_gateway.py`
|
||||||
|
- Verify: `tests/test_memory_gateway_skill.py`
|
||||||
|
|
||||||
|
- [x] **Step 1: Update the Chinese API documentation**
|
||||||
|
|
||||||
|
Document `agent_id`, `method`, `radius`, `include_profile`, `enable_llm_rerank`, `filters`, the `top_k=-1` rule, filter composition, and the `memory_type` response field. Update the curl and JSON examples with the new defaults.
|
||||||
|
|
||||||
|
- [x] **Step 2: Run formatting and full tests**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git diff --check
|
||||||
|
uv run pytest -q
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: no whitespace errors and all tests pass.
|
||||||
|
|
||||||
|
- [x] **Step 3: Review the final diff**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git diff --stat
|
||||||
|
git diff -- core/api.py core/service.py tests/test_gateway.py README.md
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: changes are limited to the approved search compatibility scope and documentation.
|
||||||
@ -57,11 +57,13 @@ INFO: Uvicorn running on http://127.0.0.1:8010 (Press CTRL+C to quit)
|
|||||||
Request:
|
Request:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
USER_ID="gateway_user_20260611180257"
|
||||||
|
|
||||||
curl -sS --location 'http://127.0.0.1:8010/users' \
|
curl -sS --location 'http://127.0.0.1:8010/users' \
|
||||||
--header 'Content-Type: application/json' \
|
--header 'Content-Type: application/json' \
|
||||||
--data '{
|
--data "{
|
||||||
"user_id": "gateway_user_20260611180257"
|
\"user_id\": \"${USER_ID}\"
|
||||||
}'
|
}"
|
||||||
```
|
```
|
||||||
|
|
||||||
Response:
|
Response:
|
||||||
@ -88,7 +90,6 @@ Request:
|
|||||||
```bash
|
```bash
|
||||||
cd /home/tom/memory-gateway
|
cd /home/tom/memory-gateway
|
||||||
|
|
||||||
USER_ID="gateway_user_20260611180257"
|
|
||||||
USER_KEY="uk_REDACTED"
|
USER_KEY="uk_REDACTED"
|
||||||
CONVERSATION_ID="gateway-multimodal-20260611180257"
|
CONVERSATION_ID="gateway-multimodal-20260611180257"
|
||||||
SESSION_ID="chat:${CONVERSATION_ID}"
|
SESSION_ID="chat:${CONVERSATION_ID}"
|
||||||
@ -160,13 +161,13 @@ Request:
|
|||||||
```bash
|
```bash
|
||||||
curl -sS --location 'http://127.0.0.1:8010/memories/flush' \
|
curl -sS --location 'http://127.0.0.1:8010/memories/flush' \
|
||||||
--header 'Content-Type: application/json' \
|
--header 'Content-Type: application/json' \
|
||||||
--data '{
|
--data "{
|
||||||
"user_id": "gateway_user_20260611180257",
|
\"user_id\": \"${USER_ID}\",
|
||||||
"user_key": "uk_REDACTED",
|
\"user_key\": \"${USER_KEY}\",
|
||||||
"session_id": "chat:gateway-multimodal-20260611180257",
|
\"session_id\": \"${SESSION_ID}\",
|
||||||
"app_id": "default",
|
\"app_id\": \"default\",
|
||||||
"project_id": "default"
|
\"project_id\": \"default\"
|
||||||
}'
|
}"
|
||||||
```
|
```
|
||||||
|
|
||||||
Response:
|
Response:
|
||||||
@ -201,16 +202,16 @@ sleep 2
|
|||||||
|
|
||||||
curl -sS --location 'http://127.0.0.1:8010/memories/search' \
|
curl -sS --location 'http://127.0.0.1:8010/memories/search' \
|
||||||
--header 'Content-Type: application/json' \
|
--header 'Content-Type: application/json' \
|
||||||
--data '{
|
--data "{
|
||||||
"user_id": "gateway_user_20260611180257",
|
\"user_id\": \"${USER_ID}\",
|
||||||
"user_key": "uk_REDACTED",
|
\"user_key\": \"${USER_KEY}\",
|
||||||
"conversation_id": "gateway-multimodal-20260611180257",
|
\"conversation_id\": \"${CONVERSATION_ID}\",
|
||||||
"query": "图片里的蓝色圆形在哪里?音频是什么?",
|
\"query\": \"图片里的蓝色圆形在哪里?音频是什么?\",
|
||||||
"scope": ["current_chat"],
|
\"scope\": [\"current_chat\"],
|
||||||
"top_k": 5,
|
\"top_k\": 5,
|
||||||
"app_id": "default",
|
\"app_id\": \"default\",
|
||||||
"project_id": "default"
|
\"project_id\": \"default\"
|
||||||
}'
|
}"
|
||||||
```
|
```
|
||||||
|
|
||||||
Response:
|
Response:
|
||||||
|
|||||||
@ -21,6 +21,7 @@ class FakeBackendClient:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
search_results: list[dict[str, Any]] | None = None,
|
search_results: list[dict[str, Any]] | None = None,
|
||||||
|
search_data: dict[str, list[dict[str, Any]]] | None = None,
|
||||||
health_error: Exception | None = None,
|
health_error: Exception | None = None,
|
||||||
add_failures: int = 0,
|
add_failures: int = 0,
|
||||||
flush_failures: int = 0,
|
flush_failures: int = 0,
|
||||||
@ -29,6 +30,7 @@ class FakeBackendClient:
|
|||||||
self.flush_calls: list[dict[str, str]] = []
|
self.flush_calls: list[dict[str, str]] = []
|
||||||
self.search_calls: list[dict[str, Any]] = []
|
self.search_calls: list[dict[str, Any]] = []
|
||||||
self.search_results = search_results or []
|
self.search_results = search_results or []
|
||||||
|
self.search_data = search_data
|
||||||
self.health_error = health_error
|
self.health_error = health_error
|
||||||
self.add_failures = add_failures
|
self.add_failures = add_failures
|
||||||
self.flush_failures = flush_failures
|
self.flush_failures = flush_failures
|
||||||
@ -56,7 +58,8 @@ class FakeBackendClient:
|
|||||||
|
|
||||||
async def search_memory(self, payload: dict[str, Any]) -> dict[str, Any]:
|
async def search_memory(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
self.search_calls.append(payload)
|
self.search_calls.append(payload)
|
||||||
return {"request_id": "search", "data": {"episodes": self.search_results}}
|
data = self.search_data or {"episodes": self.search_results}
|
||||||
|
return {"request_id": "search", "data": data}
|
||||||
|
|
||||||
async def health_check(self) -> dict[str, Any]:
|
async def health_check(self) -> dict[str, Any]:
|
||||||
if self.health_error is not None:
|
if self.health_error is not None:
|
||||||
@ -115,6 +118,75 @@ def create_test_resource(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_attachment_repository_deduplicates_and_lists_by_user_session(
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
values = {
|
||||||
|
"user_id": "u_123",
|
||||||
|
"app_id": "default",
|
||||||
|
"project_id": "default",
|
||||||
|
"session_id": "chat:c_1",
|
||||||
|
"resource_id": None,
|
||||||
|
"content_type": "image",
|
||||||
|
"name": "picture.png",
|
||||||
|
"internal_uri": "file:///private/picture.png",
|
||||||
|
"source": "memory_add_uri",
|
||||||
|
"sha256": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
first = repo.create_attachment(**values)
|
||||||
|
second = repo.create_attachment(**values)
|
||||||
|
|
||||||
|
assert second["id"] == first["id"]
|
||||||
|
assert repo.list_attachments_for_session("u_123", "chat:c_1") == [first]
|
||||||
|
assert repo.list_attachments_for_session("other", "chat:c_1") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_soft_delete_resource_also_soft_deletes_attachments(
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
create_test_resource(repo, resource_id="r_1", user_id="u_123")
|
||||||
|
repo.create_attachment(
|
||||||
|
user_id="u_123",
|
||||||
|
app_id="default",
|
||||||
|
project_id="default",
|
||||||
|
session_id="resource:u_123:r_1",
|
||||||
|
resource_id="r_1",
|
||||||
|
content_type="text",
|
||||||
|
name="a.txt",
|
||||||
|
internal_uri="file:///private/a.txt",
|
||||||
|
source="resource_upload",
|
||||||
|
sha256="sha-r_1",
|
||||||
|
)
|
||||||
|
|
||||||
|
repo.soft_delete_resource("r_1", "u_123")
|
||||||
|
|
||||||
|
assert repo.list_attachments_for_session("u_123", "resource:u_123:r_1") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_soft_delete_resource_does_not_delete_other_users_attachments(
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
create_test_resource(repo, resource_id="r_1", user_id="alice")
|
||||||
|
repo.create_attachment(
|
||||||
|
user_id="alice",
|
||||||
|
app_id="default",
|
||||||
|
project_id="default",
|
||||||
|
session_id="resource:alice:r_1",
|
||||||
|
resource_id="r_1",
|
||||||
|
content_type="text",
|
||||||
|
name="a.txt",
|
||||||
|
internal_uri="file:///private/a.txt",
|
||||||
|
source="resource_upload",
|
||||||
|
sha256="sha-r_1",
|
||||||
|
)
|
||||||
|
|
||||||
|
repo.soft_delete_resource("r_1", "bob")
|
||||||
|
|
||||||
|
attachments = repo.list_attachments_for_session("alice", "resource:alice:r_1")
|
||||||
|
assert len(attachments) == 1
|
||||||
|
|
||||||
|
|
||||||
async def create_user(client: httpx.AsyncClient, user_id: str = "u_123") -> str:
|
async def create_user(client: httpx.AsyncClient, user_id: str = "u_123") -> str:
|
||||||
response = await client.post("/users", json={"user_id": user_id})
|
response = await client.post("/users", json={"user_id": user_id})
|
||||||
assert response.status_code == 200, response.text
|
assert response.status_code == 200, response.text
|
||||||
@ -343,6 +415,31 @@ async def test_upload_binary_resource_sends_base64_content_to_backend(
|
|||||||
assert "uri" not in content
|
assert "uri" not in content
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_upload_resource_creates_attachment_mapping(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/resources",
|
||||||
|
data={"user_id": "u_123", "user_key": user_key},
|
||||||
|
files={"file": ("picture.png", b"png bytes", "image/png")},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
session_id = response.json()["session_id"]
|
||||||
|
attachments = repo.list_attachments_for_session("u_123", session_id)
|
||||||
|
assert len(attachments) == 1
|
||||||
|
assert attachments[0]["resource_id"] == response.json()["resource_id"]
|
||||||
|
assert attachments[0]["content_type"] == "image"
|
||||||
|
assert attachments[0]["name"] == "picture.png"
|
||||||
|
assert attachments[0]["internal_uri"].startswith("file://")
|
||||||
|
assert attachments[0]["source"] == "resource_upload"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_upload_resource_uses_current_timestamp(
|
async def test_upload_resource_uses_current_timestamp(
|
||||||
config: GatewayConfig,
|
config: GatewayConfig,
|
||||||
@ -607,6 +704,129 @@ async def test_add_memory_forwards_multimodal_payload_to_backend(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_memory_creates_uri_attachment_mapping(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
uri = "file:///home/tom/memory-gateway/tests/simple-multimodal-image.png"
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/add",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"session_id": "chat:c_uri",
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"sender_id": "u_123",
|
||||||
|
"role": "user",
|
||||||
|
"timestamp": 1234567890123,
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"uri": uri,
|
||||||
|
"name": "simple-multimodal-image.png",
|
||||||
|
"ext": "png",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
attachments = repo.list_attachments_for_session("u_123", "chat:c_uri")
|
||||||
|
assert [(item["name"], item["internal_uri"], item["source"]) for item in attachments] == [
|
||||||
|
("simple-multimodal-image.png", uri, "memory_add_uri")
|
||||||
|
]
|
||||||
|
assert backend.add_calls[0]["messages"][0]["content"][0]["uri"] == uri
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_memory_materializes_base64_attachment(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
encoded = base64.b64encode(b"wav bytes").decode("ascii")
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/add",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"session_id": "chat:c_base64",
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"sender_id": "u_123",
|
||||||
|
"role": "user",
|
||||||
|
"timestamp": 1234567890123,
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "audio",
|
||||||
|
"base64": encoded,
|
||||||
|
"name": "tone.wav",
|
||||||
|
"ext": "wav",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
attachments = repo.list_attachments_for_session("u_123", "chat:c_base64")
|
||||||
|
assert len(attachments) == 1
|
||||||
|
attachment = attachments[0]
|
||||||
|
assert attachment["name"] == "tone.wav"
|
||||||
|
assert attachment["source"] == "memory_add_base64"
|
||||||
|
path = Path(attachment["internal_uri"].removeprefix("file://"))
|
||||||
|
assert path.read_bytes() == b"wav bytes"
|
||||||
|
assert backend.add_calls[0]["messages"][0]["content"][0]["base64"] == encoded
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_memory_deduplicates_retried_base64_attachment(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
encoded = base64.b64encode(b"same bytes").decode("ascii")
|
||||||
|
payload: dict[str, Any] = {
|
||||||
|
"user_id": "u_123",
|
||||||
|
"session_id": "chat:c_retry",
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"sender_id": "u_123",
|
||||||
|
"role": "user",
|
||||||
|
"timestamp": 1234567890123,
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"base64": encoded,
|
||||||
|
"name": "same.png",
|
||||||
|
"ext": "png",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
payload["user_key"] = user_key
|
||||||
|
first = await client.post("/memories/add", json=payload)
|
||||||
|
second = await client.post("/memories/add", json=payload)
|
||||||
|
|
||||||
|
assert first.status_code == 200, first.text
|
||||||
|
assert second.status_code == 200, second.text
|
||||||
|
attachments = repo.list_attachments_for_session("u_123", "chat:c_retry")
|
||||||
|
assert len(attachments) == 1
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_flush_memory_forwards_request_to_backend(
|
async def test_flush_memory_forwards_request_to_backend(
|
||||||
config: GatewayConfig,
|
config: GatewayConfig,
|
||||||
@ -639,6 +859,313 @@ async def test_flush_memory_forwards_request_to_backend(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_search_forwards_default_upstream_options(
|
||||||
|
config: GatewayConfig,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/search",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"query": "hello",
|
||||||
|
"scope": ["all_user_memory"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert backend.search_calls == [
|
||||||
|
{
|
||||||
|
"user_id": "u_123",
|
||||||
|
"query": "hello",
|
||||||
|
"method": "hybrid",
|
||||||
|
"top_k": 8,
|
||||||
|
"include_profile": True,
|
||||||
|
"enable_llm_rerank": True,
|
||||||
|
"app_id": "default",
|
||||||
|
"project_id": "default",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_search_forwards_all_upstream_options(
|
||||||
|
config: GatewayConfig,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/search",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"agent_id": "agent_456",
|
||||||
|
"query": "hello",
|
||||||
|
"scope": ["all_user_memory"],
|
||||||
|
"method": "keyword",
|
||||||
|
"top_k": -1,
|
||||||
|
"radius": 0.4,
|
||||||
|
"include_profile": False,
|
||||||
|
"enable_llm_rerank": False,
|
||||||
|
"app_id": "app_1",
|
||||||
|
"project_id": "project_1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert backend.search_calls == [
|
||||||
|
{
|
||||||
|
"agent_id": "agent_456",
|
||||||
|
"query": "hello",
|
||||||
|
"method": "keyword",
|
||||||
|
"top_k": -1,
|
||||||
|
"radius": 0.4,
|
||||||
|
"include_profile": False,
|
||||||
|
"enable_llm_rerank": False,
|
||||||
|
"app_id": "app_1",
|
||||||
|
"project_id": "project_1",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "value"),
|
||||||
|
[
|
||||||
|
("method", "invalid"),
|
||||||
|
("radius", 1.1),
|
||||||
|
("top_k", 0),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_search_rejects_invalid_upstream_options(
|
||||||
|
config: GatewayConfig,
|
||||||
|
field: str,
|
||||||
|
value: Any,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
payload = {
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"query": "hello",
|
||||||
|
"scope": ["all_user_memory"],
|
||||||
|
field: value,
|
||||||
|
}
|
||||||
|
response = await client.post("/memories/search", json=payload)
|
||||||
|
|
||||||
|
assert response.status_code == 422, response.text
|
||||||
|
assert backend.search_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_search_combines_custom_and_scope_filters(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
create_test_resource(repo, resource_id="r_1", user_id="u_123")
|
||||||
|
backend = FakeBackendClient()
|
||||||
|
custom_filters = {"OR": [{"type": "Conversation"}, {"sender_ids": "u_123"}]}
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/search",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"query": "hello",
|
||||||
|
"scope": ["resources"],
|
||||||
|
"filters": custom_filters,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert backend.search_calls[0]["filters"] == {
|
||||||
|
"AND": [
|
||||||
|
custom_filters,
|
||||||
|
{"session_id": {"in": ["resource:u_123:r_1"]}},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_search_labels_all_memory_types(
|
||||||
|
config: GatewayConfig,
|
||||||
|
) -> None:
|
||||||
|
backend = FakeBackendClient(
|
||||||
|
search_data={
|
||||||
|
"episodes": [{"id": "ep_1", "session_id": "chat:c_1", "episode": "e"}],
|
||||||
|
"profiles": [{"id": "profile_1", "profile_data": {"name": "Tom"}}],
|
||||||
|
"agent_cases": [
|
||||||
|
{"id": "case_1", "session_id": "chat:c_1", "task_intent": "case"}
|
||||||
|
],
|
||||||
|
"agent_skills": [{"id": "skill_1", "content": "skill"}],
|
||||||
|
"unprocessed_messages": [
|
||||||
|
{"id": "message_1", "session_id": "chat:c_1", "content": "pending"}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/search",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"query": "hello",
|
||||||
|
"scope": ["all_user_memory"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert [item["memory_type"] for item in response.json()["results"]] == [
|
||||||
|
"episode",
|
||||||
|
"profile",
|
||||||
|
"agent_case",
|
||||||
|
"agent_skill",
|
||||||
|
"unprocessed_message",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_search_returns_attachment_when_raw_contains_filename(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
repo.create_attachment(
|
||||||
|
user_id="u_123",
|
||||||
|
app_id="default",
|
||||||
|
project_id="default",
|
||||||
|
session_id="chat:c_1",
|
||||||
|
resource_id=None,
|
||||||
|
content_type="image",
|
||||||
|
name="Picture.PNG",
|
||||||
|
internal_uri="file:///private/Picture.PNG",
|
||||||
|
source="memory_add_uri",
|
||||||
|
sha256=None,
|
||||||
|
)
|
||||||
|
backend = FakeBackendClient(
|
||||||
|
[{"id": "ep_1", "session_id": "chat:c_1", "episode": "Saw picture.png"}]
|
||||||
|
)
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/search",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"query": "picture",
|
||||||
|
"scope": ["all_user_memory"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert response.json()["results"][0]["attachments"] == [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"name": "Picture.PNG",
|
||||||
|
"internal_uri": "file:///private/Picture.PNG",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_search_omits_unmentioned_and_base64_only_attachments(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
repo.create_attachment(
|
||||||
|
user_id="u_123",
|
||||||
|
app_id="default",
|
||||||
|
project_id="default",
|
||||||
|
session_id="chat:c_1",
|
||||||
|
resource_id=None,
|
||||||
|
content_type="audio",
|
||||||
|
name="tone.wav",
|
||||||
|
internal_uri="file:///private/tone.wav",
|
||||||
|
source="memory_add_base64",
|
||||||
|
sha256=None,
|
||||||
|
)
|
||||||
|
backend = FakeBackendClient(
|
||||||
|
search_data={
|
||||||
|
"unprocessed_messages": [
|
||||||
|
{
|
||||||
|
"id": "message_1",
|
||||||
|
"session_id": "chat:c_1",
|
||||||
|
"content": [{"base64": "encoded-prefix-tone.wav"}],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client)
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/search",
|
||||||
|
json={
|
||||||
|
"user_id": "u_123",
|
||||||
|
"user_key": user_key,
|
||||||
|
"query": "audio",
|
||||||
|
"scope": ["all_user_memory"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert response.json()["results"][0]["attachments"] == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_search_attachment_mapping_is_user_isolated(
|
||||||
|
config: GatewayConfig,
|
||||||
|
repo: MemoryRepository,
|
||||||
|
) -> None:
|
||||||
|
for user_id, name in (("alice", "alice.png"), ("bob", "bob.png")):
|
||||||
|
repo.create_attachment(
|
||||||
|
user_id=user_id,
|
||||||
|
app_id="default",
|
||||||
|
project_id="default",
|
||||||
|
session_id="chat:shared",
|
||||||
|
resource_id=None,
|
||||||
|
content_type="image",
|
||||||
|
name=name,
|
||||||
|
internal_uri=f"file:///private/{name}",
|
||||||
|
source="memory_add_uri",
|
||||||
|
sha256=None,
|
||||||
|
)
|
||||||
|
backend = FakeBackendClient(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "ep_1",
|
||||||
|
"session_id": "chat:shared",
|
||||||
|
"episode": "alice.png and bob.png",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
async with app_client(config, backend) as client:
|
||||||
|
user_key = await create_user(client, "alice")
|
||||||
|
response = await client.post(
|
||||||
|
"/memories/search",
|
||||||
|
json={
|
||||||
|
"user_id": "alice",
|
||||||
|
"user_key": user_key,
|
||||||
|
"query": "images",
|
||||||
|
"scope": ["all_user_memory"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert response.json()["results"][0]["attachments"] == [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"name": "alice.png",
|
||||||
|
"internal_uri": "file:///private/alice.png",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_deleted_resource_is_excluded_from_resource_scope_search(
|
async def test_deleted_resource_is_excluded_from_resource_scope_search(
|
||||||
config: GatewayConfig,
|
config: GatewayConfig,
|
||||||
|
|||||||
Reference in New Issue
Block a user