tomtan/memory-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add multipart memory uploads

Browse Source
This commit is contained in:
tomtan
2026-06-22 15:53:29 +08:00
parent 12c767cd68
commit f77454b4cc
6 changed files with 1076 additions and 1255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1152
README.md
View File

File diff suppressed because it is too large Load Diff

83
core/api.py
View File

@ -9,7 +9,9 @@ from urllib.parse import parse_qsl, quote, urlsplit, urlunsplit
import httpx import httpx
from fastapi import APIRouter, FastAPI, File, Form, HTTPException, Request, UploadFile from fastapi import APIRouter, FastAPI, File, Form, HTTPException, Request, UploadFile
from pydantic import ValidationError
from pydantic import BaseModel, Field, field_validator from pydantic import BaseModel, Field, field_validator
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.responses import Response from starlette.responses import Response
from .config import GatewayConfig from .config import GatewayConfig
@ -220,6 +222,60 @@ def _backend_http_error_detail(exc: httpx.HTTPStatusError) -> Any:
return exc.response.text return exc.response.text
def _form_text(form: Any, field: str, default: str | None = None) -> str:
value = form.get(field)
if value is None:
if default is not None:
return default
raise HTTPException(status_code=422, detail=f"missing form field: {field}")
if isinstance(value, StarletteUploadFile):
raise HTTPException(status_code=422, detail=f"form field must be text: {field}")
return str(value)
async def _form_json_text(form: Any, field: str) -> str:
value = form.get(field)
if value is None:
raise HTTPException(status_code=422, detail=f"missing form field: {field}")
if isinstance(value, StarletteUploadFile):
raw = await value.read()
return raw.decode("utf-8")
return str(value)
def _upload_files_from_form(form: Any) -> dict[str, UploadFile]:
files: dict[str, UploadFile] = {}
for key, value in form.multi_items():
if not isinstance(value, StarletteUploadFile):
continue
if key == "messages":
continue
if key in files:
raise HTTPException(
status_code=422,
detail=f"duplicate upload file field: {key}",
)
files[key] = value
return files
async def _multipart_messages(form: Any) -> list[dict[str, Any]]:
raw = await _form_json_text(form, "messages")
try:
parsed = json.loads(raw)
except json.JSONDecodeError as exc:
raise HTTPException(
status_code=400,
detail=f"invalid messages JSON: {exc.msg}",
) from exc
if not isinstance(parsed, list):
raise HTTPException(status_code=400, detail="messages must be a JSON array")
try:
return [AddMemoryMessage.model_validate(item).model_dump() for item in parsed]
except ValidationError as exc:
raise HTTPException(status_code=422, detail=exc.errors()) from exc
def create_app( def create_app(
*, *,
config: GatewayConfig | None = None, config: GatewayConfig | None = None,
@ -466,6 +522,33 @@ def create_app(
except InvalidAttachment as exc: except InvalidAttachment as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc raise HTTPException(status_code=422, detail=str(exc)) from exc
@router.post("/memories/add/multipart")
async def add_memory_multipart(request: Request) -> dict[str, Any]:
form = await request.form()
user_id = _form_text(form, "user_id")
user_key = _form_text(form, "user_key")
require_user(user_id, user_key)
try:
return await service.add_memory_with_uploads(
user_id=user_id,
session_id=_form_text(form, "session_id"),
app_id=_form_text(form, "app_id", "default"),
project_id=_form_text(form, "project_id", "default"),
messages=await _multipart_messages(form),
upload_files=_upload_files_from_form(form),
)
except httpx.HTTPStatusError as exc:
raise HTTPException(
status_code=exc.response.status_code,
detail=_backend_http_error_detail(exc),
) from exc
except UploadTooLarge as exc:
raise HTTPException(status_code=413, detail=str(exc)) from exc
except UnsupportedContentType as exc:
raise HTTPException(status_code=415, detail=str(exc)) from exc
except InvalidAttachment as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@router.post("/memories/flush") @router.post("/memories/flush")
async def flush_memory( async def flush_memory(
request: FlushMemoryRequest, request: FlushMemoryRequest,

167
core/service.py
View File

@ -138,6 +138,25 @@ def _remove_empty_parents(path: Path, stop_at: Path | None = None) -> None:
current = parent current = parent
def _read_upload_bytes(
file: UploadFile,
max_upload_bytes: int,
) -> tuple[bytes, str, int]:
sha256 = hashlib.sha256()
size = 0
chunks: list[bytes] = []
while True:
chunk = file.file.read(1024 * 1024)
if not chunk:
break
size += len(chunk)
if size > max_upload_bytes:
raise UploadTooLarge(f"upload exceeds max size of {max_upload_bytes} bytes")
sha256.update(chunk)
chunks.append(chunk)
return b"".join(chunks), sha256.hexdigest(), size
class MemoryGatewayService: class MemoryGatewayService:
def __init__( def __init__(
self, self,
@ -617,6 +636,41 @@ class MemoryGatewayService:
raise raise
return {"session_id": session_id, "backend": backend} return {"session_id": session_id, "backend": backend}
async def add_memory_with_uploads(
self,
*,
user_id: str,
session_id: str,
app_id: str,
project_id: str,
messages: list[dict[str, Any]],
upload_files: dict[str, UploadFile],
) -> dict[str, Any]:
messages, attachments, generated_paths = self._prepare_uploaded_memory_files(
user_id=user_id,
session_id=session_id,
app_id=app_id,
project_id=project_id,
messages=messages,
upload_files=upload_files,
)
payload = {
"session_id": session_id,
"app_id": app_id,
"project_id": project_id,
"messages": messages,
}
try:
backend = await self.backend_client.add_memory(payload)
for attachment in attachments:
self.repository.create_attachment(**attachment)
except Exception:
for path in generated_paths:
path.unlink(missing_ok=True)
_remove_empty_parents(path.parent, stop_at=self.config.storage_dir)
raise
return {"session_id": session_id, "backend": backend}
def _register_resource_attachment( def _register_resource_attachment(
self, self,
resource: dict[str, Any], resource: dict[str, Any],
@ -713,6 +767,119 @@ class MemoryGatewayService:
raise raise
return attachments, generated_paths return attachments, generated_paths
def _prepare_uploaded_memory_files(
self,
*,
user_id: str,
session_id: str,
app_id: str,
project_id: str,
messages: list[dict[str, Any]],
upload_files: dict[str, UploadFile],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[Path]]:
attachments: list[dict[str, Any]] = []
generated_paths: list[Path] = []
used_upload_ids: set[str] = set()
try:
for message in messages:
content = message.get("content")
if not isinstance(content, list):
continue
for index, item in enumerate(content):
if not isinstance(item, dict) or "upload_id" not in item:
continue
upload_id = str(item.get("upload_id") or "").strip()
if not upload_id:
raise InvalidAttachment("upload_id must not be empty")
if upload_id in used_upload_ids:
raise InvalidAttachment(f"duplicate upload_id: {upload_id}")
file = upload_files.get(upload_id)
if file is None:
raise InvalidAttachment(
f"missing upload file for upload_id: {upload_id}"
)
used_upload_ids.add(upload_id)
content[index] = self._materialize_uploaded_content_item(
user_id=user_id,
session_id=session_id,
app_id=app_id,
project_id=project_id,
item=item,
file=file,
attachments=attachments,
generated_paths=generated_paths,
)
unused_upload_ids = sorted(set(upload_files) - used_upload_ids)
if unused_upload_ids:
raise InvalidAttachment(
f"unused upload file field: {unused_upload_ids[0]}"
)
except Exception:
for path in generated_paths:
path.unlink(missing_ok=True)
_remove_empty_parents(path.parent, stop_at=self.config.storage_dir)
raise
return messages, attachments, generated_paths
def _materialize_uploaded_content_item(
self,
*,
user_id: str,
session_id: str,
app_id: str,
project_id: str,
item: dict[str, Any],
file: UploadFile,
attachments: list[dict[str, Any]],
generated_paths: list[Path],
) -> dict[str, Any]:
name = _safe_filename(str(item.get("name") or file.filename or "upload.bin"))
mime_type = file.content_type or mimetypes.guess_type(name)[0]
if not _mime_allowed(mime_type, self.config.allowed_mime_types):
raise UnsupportedContentType(f"unsupported content type: {mime_type}")
content_type = normalize_content_type(
name,
mime_type,
str(item.get("type") or ""),
)
data, sha256, _size_bytes = _read_upload_bytes(
file,
self.config.max_upload_bytes,
)
path = self.config.storage_dir / user_id / "memory_attachments" / sha256 / name
if not path.exists():
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
generated_paths.append(path)
content_item = {
key: value for key, value in item.items() if key not in {"upload_id", "uri"}
}
content_item["type"] = content_type
content_item["name"] = name
content_item["ext"] = Path(name).suffix.lstrip(".") or content_item.get("ext")
if content_type == "text":
content_item.pop("base64", None)
content_item["text"] = data.decode("utf-8", errors="replace")
else:
content_item.pop("text", None)
content_item["base64"] = base64.b64encode(data).decode("ascii")
attachments.append(
{
"id": f"a_{uuid.uuid4().hex}",
"user_id": user_id,
"app_id": app_id,
"project_id": project_id,
"session_id": session_id,
"resource_id": None,
"content_type": content_type,
"name": name,
"internal_uri": path.resolve().as_uri(),
"source": "memory_add_upload",
"sha256": sha256,
}
)
return content_item
async def flush_memory( async def flush_memory(
self, self,
*, *,

39
skill/memory-gateway-agent/references/api.md
View File

@ -136,6 +136,43 @@ $CLI flush-memory --session-id chat:c_456
`--messages` accepts either a JSON array string or a path to a JSON file. Always flush after all messages for the session have been added. `--messages` accepts either a JSON array string or a path to a JSON file. Always flush after all messages for the session have been added.
For local binary files that cannot be converted to base64 by the caller, use the
multipart API directly. Put an `upload_id` in the content item and send a file
field with the same name:
```bash
curl -X POST "$MEMORY_GATEWAY_BASE_URL/memories/add/multipart" \
-F user_id="$MEMORY_GATEWAY_USER_ID" \
-F user_key="$MEMORY_GATEWAY_USER_KEY" \
-F session_id=chat:c_456 \
-F app_id=default \
-F project_id=default \
-F 'messages=[
{
"sender_id": "u_123",
"role": "user",
"timestamp": 1781172177000,
"content": [
{"type": "text", "text": "Remember this image"},
{
"type": "image",
"upload_id": "image_1",
"name": "image.png",
"ext": "png"
}
]
}
]' \
-F 'image_1=@./image.png;type=image/png'
```
The multipart endpoint appends messages to the provided chat session. It stores
the uploaded file under Gateway storage, forwards text/base64 content to the
upstream memory service, and records an attachment mapping. Call `flush-memory`
afterward when the session should be extracted and indexed. This differs from
`upload-resource`, which creates an independent `resource:{user_id}:{resource_id}`
session and automatically performs add plus flush for resource searches.
### Override and Delete Memory ### Override and Delete Memory
Use IDs from a search result: Use IDs from a search result:
@ -188,6 +225,8 @@ Common content items:
``` ```
Prefer base64 for local binary files. A `file://` URI is only usable when upstream memory service can access the same filesystem path. Prefer base64 for local binary files. A `file://` URI is only usable when upstream memory service can access the same filesystem path.
If that shared path guarantee is not true, use `/memories/add/multipart`,
`upload-resource`, or `/resources/external`.
## Search Scopes ## Search Scopes

779
tests/test_command.md
View File

@ -1,144 +1,126 @@
# Memory Gateway multimodal API test # Memory Gateway API curl examples
This file records a real end-to-end test through **Memory Gateway**, not direct upstream memory service calls. This file keeps only the concrete API curl shapes and short notes. Replace
`<USER_KEY>` with the key returned by `POST /users`.
Gateway URL used by curl: Base URL used in the live deployment test:
```text ```text
http://127.0.0.1:8010 http://127.0.0.1:8010
``` ```
Gateway upstream memory service: Test files:
```text ```text
http://10.6.80.123:1995 tests/simple-multimodal-image.png
tests/simple-tone.wav
``` ```
Test assets: ## 1. Health
```text
/home/tom/memory-gateway/tests/simple-multimodal-image.png
/home/tom/memory-gateway/tests/simple-tone.wav
```
Asset check:
```text
tests/simple-multimodal-image.png: PNG image data, 96 x 64, 8-bit/color RGB, non-interlaced
tests/simple-tone.wav: RIFF (little-endian) data, WAVE audio, Microsoft PCM, 16 bit, mono 8000 Hz
```
## Start Gateway
Command:
```bash ```bash
cd /home/tom/memory-gateway curl -sS http://127.0.0.1:8010/health
MEMORY_GATEWAY_BACKEND_BASE_URL=http://10.6.80.123:1995 \
MEMORY_GATEWAY_DB_PATH=/tmp/memory_gateway_curl.sqlite3 \
MEMORY_GATEWAY_STORAGE_DIR=/tmp/memory_gateway_curl_storage \
MEMORY_GATEWAY_HOST=127.0.0.1 \
MEMORY_GATEWAY_PORT=8010 \
.venv/bin/python main.py
``` ```
Observed startup: Expected shape:
```text
INFO: Started server process [771099]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on http://127.0.0.1:8010 (Press CTRL+C to quit)
```
## 1. Create Gateway user
Request:
```bash
USER_ID="gateway_user_20260611180257"
curl -sS --location 'http://127.0.0.1:8010/users' \
--header 'Content-Type: application/json' \
--data "{
\"user_id\": \"${USER_ID}\"
}"
```
Response:
```json ```json
{ {
"user_id": "gateway_user_20260611180257", "status": "ok",
"user_key": "uk_REDACTED", "api": {"status": "ok"},
"created_at": "2026-06-11T10:02:57.435437+00:00" "backend": {
"status": "ok",
"base_url": "http://0.0.0.0:1995",
"data": {"status": "ok"}
}
} }
``` ```
HTTP metadata: ## 2. Create user
```text ```bash
HTTP_STATUS:200 curl -sS -X POST http://127.0.0.1:8010/users \
TOTAL_TIME:0.022431 -H 'Content-Type: application/json' \
-d '{"user_id":"gateway_demo_user"}'
``` ```
## 2. Add text + audio(base64) + image(file) through Gateway Expected shape:
```json
{
"user_id": "gateway_demo_user",
"user_key": "uk_REDACTED",
"created_at": "2026-06-22T06:54:35.823262+00:00"
}
```
Use the returned `user_key` in later requests.
## 3. Add chat memory with multipart files
Use this when files belong to a chat/session message and the client should not
or cannot convert the files to base64.
`upload_id` rules:
- `upload_id` is defined by the caller.
- Gateway does not generate it.
- Gateway does not require a format such as `user_id_filetype_number`.
- It only needs to be non-empty, unique inside the request, and equal to the
multipart file field name.
- Good simple values are `image_1`, `image_2`, `audio_1`, `doc_1`.
In the `messages` JSON, `upload_id: "image_1"` points to this file field:
```bash
-F 'image_1=@tests/simple-multimodal-image.png;type=image/png'
```
Request: Request:
```bash ```bash
cd /home/tom/memory-gateway curl -sS -X POST http://127.0.0.1:8010/memories/add/multipart \
-F 'user_id=gateway_demo_user' \
USER_KEY="uk_REDACTED" -F 'user_key=<USER_KEY>' \
CONVERSATION_ID="gateway-multimodal-20260611180257" -F 'session_id=chat:gateway_demo_conversation' \
SESSION_ID="chat:${CONVERSATION_ID}" -F 'app_id=default' \
TIMESTAMP_MS="1781172177000" -F 'project_id=default' \
AUDIO_BASE64="$(base64 -w0 tests/simple-tone.wav)" -F 'messages=[
{
curl -sS --location 'http://127.0.0.1:8010/memories/add' \ "sender_id": "gateway_demo_user",
--header 'Content-Type: application/json' \ "role": "user",
--data "{ "timestamp": 1782111275810,
\"user_id\": \"${USER_ID}\", "content": [
\"user_key\": \"${USER_KEY}\", {
\"session_id\": \"${SESSION_ID}\", "type": "text",
\"app_id\": \"default\", "text": "请记住这次上传：图片里有左上红色方块、右上蓝色圆形、底部绿色横条；音频是一段短促测试音。"
\"project_id\": \"default\", },
\"messages\": [ {
{ "type": "image",
\"sender_id\": \"${USER_ID}\", "upload_id": "image_1",
\"role\": \"user\", "name": "simple-multimodal-image.png",
\"timestamp\": ${TIMESTAMP_MS}, "ext": "png"
\"content\": [ },
{ {
\"type\": \"text\", "type": "audio",
\"text\": \"请通过 Memory Gateway 同时记住这段文字、音频和图片：图片里有左侧红色方块、右侧蓝色圆形、底部绿色横条；音频是一段短促的测试音。以后可能会问图片中各个物体的位置和颜色。\" "upload_id": "audio_1",
}, "name": "simple-tone.wav",
{ "ext": "wav"
\"type\": \"audio\", }
\"base64\": \"${AUDIO_BASE64}\", ]
\"ext\": \"wav\", }
\"name\": \"simple-tone.wav\" ]' \
}, -F 'image_1=@tests/simple-multimodal-image.png;type=image/png' \
{ -F 'audio_1=@tests/simple-tone.wav;type=audio/wav'
\"type\": \"image\",
\"uri\": \"file:///home/tom/memory-gateway/tests/simple-multimodal-image.png\",
\"ext\": \"png\",
\"name\": \"simple-multimodal-image.png\"
}
]
}
]
}"
``` ```
Response: Expected shape:
```json ```json
{ {
"session_id": "chat:gateway-multimodal-20260611180257", "session_id": "chat:gateway_demo_conversation",
"backend": { "backend": {
"request_id": "c9e24b8d27ee4ad08a8df70273336637", "request_id": "0d6451f4077040e4af207cc6b034ea34",
"data": { "data": {
"message_count": 1, "message_count": 1,
"status": "accumulated" "status": "accumulated"
@ -147,308 +129,153 @@ Response:
} }
``` ```
HTTP metadata: Gateway stores the uploaded files and forwards upstream-compatible `base64` or
`text` content. The client does not send `file://` and does not send base64.
```text Common errors:
HTTP_STATUS:200
TOTAL_TIME:1.552665
```
## 3. Flush through Gateway - Missing file field for an `upload_id`: `422`
- Duplicate `upload_id`: `422`
- Extra uploaded file field not referenced by `messages`: `422`
- Unsupported MIME type: `415`
- File too large: `413`
Request: ## 4. Flush chat session
`/memories/add/multipart` only appends messages. Call flush when the session
should be extracted and indexed.
```bash ```bash
curl -sS --location 'http://127.0.0.1:8010/memories/flush' \ curl -sS -X POST http://127.0.0.1:8010/memories/flush \
--header 'Content-Type: application/json' \ -H 'Content-Type: application/json' \
--data "{ -d '{
\"user_id\": \"${USER_ID}\", "user_id": "gateway_demo_user",
\"user_key\": \"${USER_KEY}\", "user_key": "<USER_KEY>",
\"session_id\": \"${SESSION_ID}\", "session_id": "chat:gateway_demo_conversation",
\"app_id\": \"default\", "app_id": "default",
\"project_id\": \"default\" "project_id": "default"
}" }'
``` ```
Response: Expected shape:
```json ```json
{ {
"session_id": "chat:gateway-multimodal-20260611180257", "session_id": "chat:gateway_demo_conversation",
"backend": { "backend": {
"request_id": "8eb7d5db2d3b43f4999f445aabb813b1", "request_id": "4df5415115a34f109c564abd2f9012c6",
"data": { "data": {"status": "extracted"}
"status": "extracted"
}
} }
} }
``` ```
HTTP metadata: ## 5. Search chat session
```text
HTTP_STATUS:200
TOTAL_TIME:2.135721
```
## 4. Search through Gateway
upstream memory service indexing can lag briefly after `flush`, so this test waited about 2 seconds before searching.
Request:
```bash ```bash
sleep 2 curl -sS -X POST http://127.0.0.1:8010/memories/search \
-H 'Content-Type: application/json' \
curl -sS --location 'http://127.0.0.1:8010/memories/search' \ -d '{
--header 'Content-Type: application/json' \ "user_id": "gateway_demo_user",
--data "{ "user_key": "<USER_KEY>",
\"user_id\": \"${USER_ID}\", "conversation_id": "gateway_demo_conversation",
\"user_key\": \"${USER_KEY}\", "query": "图片里的蓝色圆形在哪里？底部是什么颜色的横条？",
\"conversation_id\": \"${CONVERSATION_ID}\", "scope": ["current_chat"],
\"query\": \"图片里的蓝色圆形在哪里？音频是什么？\", "top_k": 5,
\"scope\": [\"current_chat\"], "app_id": "default",
\"top_k\": 5, "project_id": "default"
\"app_id\": \"default\", }'
\"project_id\": \"default\"
}"
``` ```
Response: Expected result excerpt:
```json ```json
{ {
"results": [ "results": [
{ {
"id": "gateway_user_20260611180257_ep_20260611_00000001", "session_id": "chat:gateway_demo_conversation",
"session_id": "chat:gateway-multimodal-20260611180257",
"text": "On June 11, 2026 at 10:02 AM UTC, user gateway_user_20260611180257 uploaded a multimodal memory package via Memory Gateway. The package included an image file named simple-multimodal-image.png and a short test audio clip. The image displayed three geometric shapes on a light gray background: a solid red square in the upper-left, a solid blue circle in the upper-right (horizontally aligned with the square), and a long, thin green horizontal rectangle spanning the bottom below both shapes. The user instructed the system to retain these details, anticipating future queries regarding the objects' positions and colors.",
"score": 0.6069304347038269,
"source_scope": "current_chat", "source_scope": "current_chat",
"resource_id": null, "text": "The image contained a red square, a blue circle, and a green horizontal rectangle.",
"resource_uri": null, "attachments": [
"raw": { {
"id": "gateway_user_20260611180257_ep_20260611_00000001", "type": "image",
"user_id": "gateway_user_20260611180257", "name": "simple-multimodal-image.png",
"app_id": "default", "internal_uri": "file:///home/tom/memory-gateway/data/storage/..."
"project_id": "default", },
"session_id": "chat:gateway-multimodal-20260611180257", {
"timestamp": "2026-06-11T10:02:57Z", "type": "audio",
"sender_ids": [ "name": "simple-tone.wav",
"gateway_user_20260611180257" "internal_uri": "file:///home/tom/memory-gateway/data/storage/..."
], }
"summary": "On June 11, 2026 at 10:02 AM UTC, user gateway_user_20260611180257 uploaded a multimodal memory package via Memory Gateway. The package included an image file named simple-multimodal-image.png and a s", ]
"subject": "gateway_user_20260611180257 Multimodal Memory Upload June 11, 2026",
"episode": "On June 11, 2026 at 10:02 AM UTC, user gateway_user_20260611180257 uploaded a multimodal memory package via Memory Gateway. The package included an image file named simple-multimodal-image.png and a short test audio clip. The image displayed three geometric shapes on a light gray background: a solid red square in the upper-left, a solid blue circle in the upper-right (horizontally aligned with the square), and a long, thin green horizontal rectangle spanning the bottom below both shapes. The user instructed the system to retain these details, anticipating future queries regarding the objects' positions and colors.",
"type": "Conversation",
"score": 0.6069304347038269,
"atomic_facts": [
{
"id": "gateway_user_20260611180257_af_20260611_00000004",
"content": "gateway_user_20260611180257 stated that questions about the positions and colors of the objects in the image might be asked in the future.",
"score": 0.6069304347038269
}
]
}
} }
] ]
} }
``` ```
HTTP metadata: ## 6. Upload an independent resource
```text Use `/resources` when the file is an independent resource, not just an
HTTP_STATUS:200 attachment inside one chat message.
TOTAL_TIME:0.064128
```
# Other Memory Gateway API tests
The following calls used a temporary Gateway database and storage directory. All requests target Memory Gateway at `http://127.0.0.1:8010`.
## 5. Health
Request:
```bash ```bash
curl -sS --location 'http://127.0.0.1:8010/health' curl -sS -X POST http://127.0.0.1:8010/resources \
-F 'user_id=gateway_demo_user' \
-F 'user_key=<USER_KEY>' \
-F 'app_id=default' \
-F 'project_id=default' \
-F 'title=Gateway demo image resource' \
-F 'description=Demo upload for simple multimodal image' \
-F 'file=@tests/simple-multimodal-image.png;type=image/png'
``` ```
Response: Expected shape:
```json ```json
{ {
"status": "ok", "resource_id": "r_1678eacf3e8c49f9a8863454c5b35e68",
"api": {"status": "ok"}, "session_id": "resource:gateway_demo_user:r_1678eacf3e8c49f9a8863454c5b35e68",
"backend": { "uri": "resource://gateway_demo_user/r_1678eacf3e8c49f9a8863454c5b35e68",
"status": "ok",
"base_url": "http://10.6.80.123:1995",
"data": {"status": "ok"}
}
}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.034914
```
## 6. Invalid credentials
Request:
```bash
curl -sS --location \
'http://127.0.0.1:8010/resources?user_id=other_api_20260612095541&user_key=wrong-key'
```
Response:
```json
{"detail":"invalid user credentials"}
```
```text
HTTP_STATUS:401
TOTAL_TIME:0.001447
```
## 7. Upload resource
The temporary test user was created with:
```bash
curl -sS --location 'http://127.0.0.1:8010/users' \
--header 'Content-Type: application/json' \
--data '{"user_id":"other_api_20260612095541"}'
```
User response:
```json
{
"user_id": "other_api_20260612095541",
"user_key": "uk_REDACTED",
"created_at": "2026-06-12T01:55:41.448076+00:00"
}
```
Upload request:
```bash
cd /home/tom/memory-gateway
curl -sS --location 'http://127.0.0.1:8010/resources' \
--form 'user_id=other_api_20260612095541' \
--form 'user_key=uk_REDACTED' \
--form 'app_id=default' \
--form 'project_id=default' \
--form 'title=Gateway API image resource' \
--form 'description=Resource lifecycle test through Memory Gateway' \
--form 'file=@tests/simple-multimodal-image.png;type=image/png'
```
Response:
```json
{
"resource_id": "r_2700e435f72a49e6a7f736d17f8c7ac7",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"uri": "resource://other_api_20260612095541/r_2700e435f72a49e6a7f736d17f8c7ac7",
"status": "extracted" "status": "extracted"
} }
``` ```
```text Unlike `/memories/add/multipart`, `/resources` automatically calls upstream add
HTTP_STATUS:200 and flush.
TOTAL_TIME:4.700296
```
## 8. List resources ## 7. List resources
Request:
```bash ```bash
curl -sS --location \ curl -sS \
'http://127.0.0.1:8010/resources?user_id=other_api_20260612095541&user_key=uk_REDACTED' 'http://127.0.0.1:8010/resources?user_id=gateway_demo_user&user_key=<USER_KEY>'
``` ```
Response: Expected shape:
```json ```json
{ {
"resources": [ "resources": [
{ {
"resource_id": "r_2700e435f72a49e6a7f736d17f8c7ac7", "resource_id": "r_1678eacf3e8c49f9a8863454c5b35e68",
"user_id": "other_api_20260612095541",
"filename": "simple-multimodal-image.png", "filename": "simple-multimodal-image.png",
"content_type": "image", "content_type": "image",
"mime_type": "image/png", "mime_type": "image/png",
"uri": "resource://other_api_20260612095541/r_2700e435f72a49e6a7f736d17f8c7ac7", "uri": "resource://gateway_demo_user/r_1678eacf3e8c49f9a8863454c5b35e68",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7", "session_id": "resource:gateway_demo_user:r_1678eacf3e8c49f9a8863454c5b35e68",
"status": "extracted", "status": "extracted"
"title": "Gateway API image resource",
"description": "Resource lifecycle test through Memory Gateway",
"created_at": "2026-06-12T01:55:41.527716+00:00",
"updated_at": "2026-06-12T01:55:46.204082+00:00"
} }
] ]
} }
``` ```
```text ## 8. Search resources
HTTP_STATUS:200
TOTAL_TIME:0.001785
```
## 9. Resource detail
Request:
```bash ```bash
curl -sS --location \ curl -sS -X POST http://127.0.0.1:8010/memories/search \
'http://127.0.0.1:8010/resources/r_2700e435f72a49e6a7f736d17f8c7ac7?user_id=other_api_20260612095541&user_key=uk_REDACTED' -H 'Content-Type: application/json' \
``` -d '{
"user_id": "gateway_demo_user",
Response: "user_key": "<USER_KEY>",
"query": "这张资源图片里有哪些几何图形和颜色？",
```json
{
"resources": [
{
"resource_id": "r_2700e435f72a49e6a7f736d17f8c7ac7",
"user_id": "other_api_20260612095541",
"filename": "simple-multimodal-image.png",
"content_type": "image",
"mime_type": "image/png",
"uri": "resource://other_api_20260612095541/r_2700e435f72a49e6a7f736d17f8c7ac7",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"status": "extracted",
"title": "Gateway API image resource",
"description": "Resource lifecycle test through Memory Gateway",
"created_at": "2026-06-12T01:55:41.527716+00:00",
"updated_at": "2026-06-12T01:55:46.204082+00:00"
}
]
}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.001634
```
## 10. Search resource memory
Request:
```bash
curl -sS --location 'http://127.0.0.1:8010/memories/search' \
--header 'Content-Type: application/json' \
--data '{
"user_id": "other_api_20260612095541",
"user_key": "uk_REDACTED",
"query": "图片中有哪些颜色和形状？",
"scope": ["resources"], "scope": ["resources"],
"top_k": 5, "top_k": 5,
"app_id": "default", "app_id": "default",
@ -456,251 +283,43 @@ curl -sS --location 'http://127.0.0.1:8010/memories/search' \
}' }'
``` ```
Response: Expected result excerpt:
```json ```json
{ {
"results": [ "results": [
{ {
"id": "other_api_20260612095541_ep_20260612_00000001",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"text": "On June 12, 2026 at 01:55 AM UTC, the user other_api_20260612095541 uploaded an image titled 'simple-multimodal-image.png' for visual analysis. The image displayed three distinct geometric shapes on a plain, light gray background. The composition included a solid red square in the upper-left portion, a solid blue circle in the upper-right portion, and a long, thin, horizontal green rectangle situated below both shapes. The red square and blue circle were roughly aligned horizontally, while the green rectangle spanned a width greater than either of the upper shapes.",
"score": 0.6418947577476501,
"source_scope": "resources", "source_scope": "resources",
"resource_id": "r_2700e435f72a49e6a7f736d17f8c7ac7", "resource_id": "r_1678eacf3e8c49f9a8863454c5b35e68",
"resource_uri": "resource://other_api_20260612095541/r_2700e435f72a49e6a7f736d17f8c7ac7", "resource_uri": "resource://gateway_demo_user/r_1678eacf3e8c49f9a8863454c5b35e68",
"raw": { "text": "The image displayed a red square, a blue circle, and a green rectangle.",
"id": "other_api_20260612095541_ep_20260612_00000001", "attachments": [
"user_id": "other_api_20260612095541", {
"app_id": "default", "type": "image",
"project_id": "default", "name": "simple-multimodal-image.png",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7", "internal_uri": "file:///home/tom/memory-gateway/data/storage/..."
"timestamp": "2026-06-12T01:55:41.541000Z", }
"sender_ids": ["other_api_20260612095541"], ]
"summary": "On June 12, 2026 at 01:55 AM UTC, the user other_api_20260612095541 uploaded an image titled 'simple-multimodal-image.png' for visual analysis. The image displayed three distinct geometric shapes on a",
"subject": "Visual Analysis of Geometric Shapes Uploaded by other_api_20260612095541 on June 12, 2026",
"episode": "On June 12, 2026 at 01:55 AM UTC, the user other_api_20260612095541 uploaded an image titled 'simple-multimodal-image.png' for visual analysis. The image displayed three distinct geometric shapes on a plain, light gray background. The composition included a solid red square in the upper-left portion, a solid blue circle in the upper-right portion, and a long, thin, horizontal green rectangle situated below both shapes. The red square and blue circle were roughly aligned horizontally, while the green rectangle spanned a width greater than either of the upper shapes.",
"type": "Conversation",
"score": 0.6418947577476501,
"atomic_facts": [
{
"id": "other_api_20260612095541_af_20260612_00000001",
"content": "The image displays three distinct geometric shapes on a plain, light gray background.",
"score": 0.6418947577476501
}
]
}
} }
] ]
} }
``` ```
```text ## Multipart vs resources
HTTP_STATUS:200
TOTAL_TIME:0.176981
```
## 11. Override memory Use `/memories/add/multipart` when the upload belongs to a chat/session message:
Request: - caller supplies `session_id`, usually `chat:{conversation_id}`;
- caller defines `upload_id` values in `messages`;
- caller uploads files as form fields with names matching `upload_id`;
- Gateway only calls upstream add;
- caller should call `/memories/flush`;
- search normally uses `current_chat` or `all_user_memory`.
```bash Use `/resources` when the upload is an independent resource:
curl -sS --location --request PATCH \
'http://127.0.0.1:8010/memories/other_api_20260612095541_ep_20260612_00000001' \
--header 'Content-Type: application/json' \
--data '{
"user_id": "other_api_20260612095541",
"user_key": "uk_REDACTED",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"override_text": "OVERRIDE: 图片左侧是红色方块，右侧是蓝色圆形，底部是绿色横条。"
}'
```
Response: - Gateway creates `resource_id`;
- Gateway creates `session_id = resource:{user_id}:{resource_id}`;
```json - Gateway writes `user_resources`;
{ - Gateway automatically calls upstream add and flush;
"memory_id": "other_api_20260612095541_ep_20260612_00000001", - search normally uses `resources`.
"override_id": "o_328f03b40b164c4896640fd2567042cb",
"status": "active"
}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.007037
```
The next search returned the overridden text:
Request:
```bash
curl -sS --location 'http://127.0.0.1:8010/memories/search' \
--header 'Content-Type: application/json' \
--data '{
"user_id": "other_api_20260612095541",
"user_key": "uk_REDACTED",
"query": "图片中有哪些颜色和形状？",
"scope": ["resources"],
"top_k": 5,
"app_id": "default",
"project_id": "default"
}'
```
```json
{
"results": [
{
"id": "other_api_20260612095541_ep_20260612_00000001",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"text": "OVERRIDE: 图片左侧是红色方块，右侧是蓝色圆形，底部是绿色横条。",
"score": 0.6418947577476501,
"source_scope": "resources",
"resource_id": "r_2700e435f72a49e6a7f736d17f8c7ac7",
"resource_uri": "resource://other_api_20260612095541/r_2700e435f72a49e6a7f736d17f8c7ac7",
"raw": {
"id": "other_api_20260612095541_ep_20260612_00000001",
"user_id": "other_api_20260612095541",
"app_id": "default",
"project_id": "default",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"timestamp": "2026-06-12T01:55:41.541000Z",
"sender_ids": ["other_api_20260612095541"],
"summary": "On June 12, 2026 at 01:55 AM UTC, the user other_api_20260612095541 uploaded an image titled 'simple-multimodal-image.png' for visual analysis. The image displayed three distinct geometric shapes on a",
"subject": "Visual Analysis of Geometric Shapes Uploaded by other_api_20260612095541 on June 12, 2026",
"episode": "On June 12, 2026 at 01:55 AM UTC, the user other_api_20260612095541 uploaded an image titled 'simple-multimodal-image.png' for visual analysis. The image displayed three distinct geometric shapes on a plain, light gray background. The composition included a solid red square in the upper-left portion, a solid blue circle in the upper-right portion, and a long, thin, horizontal green rectangle situated below both shapes. The red square and blue circle were roughly aligned horizontally, while the green rectangle spanned a width greater than either of the upper shapes.",
"type": "Conversation",
"score": 0.6418947577476501,
"atomic_facts": [
{
"id": "other_api_20260612095541_af_20260612_00000001",
"content": "The image displays three distinct geometric shapes on a plain, light gray background.",
"score": 0.6418947577476501
}
]
},
"override_id": "o_328f03b40b164c4896640fd2567042cb"
}
]
}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.055485
```
## 12. Delete memory with tombstone
Request:
```bash
curl -sS --location --request DELETE \
'http://127.0.0.1:8010/memories/other_api_20260612095541_ep_20260612_00000001' \
--header 'Content-Type: application/json' \
--data '{
"user_id": "other_api_20260612095541",
"user_key": "uk_REDACTED",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"reason": "Gateway API tombstone test"
}'
```
Response:
```json
{
"memory_id": "other_api_20260612095541_ep_20260612_00000001",
"tombstone_id": "t_2cba49bf3b6641ea96865612deebc036",
"status": "deleted"
}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.006502
```
Repeating the resource search after creating the tombstone:
```bash
curl -sS --location 'http://127.0.0.1:8010/memories/search' \
--header 'Content-Type: application/json' \
--data '{
"user_id": "other_api_20260612095541",
"user_key": "uk_REDACTED",
"query": "图片中有哪些颜色和形状？",
"scope": ["resources"],
"top_k": 5,
"app_id": "default",
"project_id": "default"
}'
```
```json
{"results":[]}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.067841
```
## 13. Delete resource
Request:
```bash
curl -sS --location --request DELETE \
'http://127.0.0.1:8010/resources/r_2700e435f72a49e6a7f736d17f8c7ac7?user_id=other_api_20260612095541&user_key=uk_REDACTED'
```
Response:
```json
{
"resource_id": "r_2700e435f72a49e6a7f736d17f8c7ac7",
"session_id": "resource:other_api_20260612095541:r_2700e435f72a49e6a7f736d17f8c7ac7",
"uri": "resource://other_api_20260612095541/r_2700e435f72a49e6a7f736d17f8c7ac7",
"status": "deleted"
}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.014089
```
List after deletion:
```bash
curl -sS --location \
'http://127.0.0.1:8010/resources?user_id=other_api_20260612095541&user_key=uk_REDACTED'
```
```json
{"resources":[]}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.001226
```
Detail after deletion:
```bash
curl -sS --location \
'http://127.0.0.1:8010/resources/r_2700e435f72a49e6a7f736d17f8c7ac7?user_id=other_api_20260612095541&user_key=uk_REDACTED'
```
```json
{"resources":[]}
```
```text
HTTP_STATUS:200
TOTAL_TIME:0.001223
```

111
tests/test_gateway.py
View File

@ -897,6 +897,117 @@ async def test_add_memory_materializes_base64_attachment(
assert backend.add_calls[0]["messages"][0]["content"][0]["base64"] == encoded assert backend.add_calls[0]["messages"][0]["content"][0]["base64"] == encoded
@pytest.mark.asyncio
async def test_add_memory_multipart_uploads_files_to_chat_session(
config: GatewayConfig,
repo: MemoryRepository,
) -> None:
backend = FakeBackendClient()
messages = [
{
"sender_id": "u_123",
"role": "user",
"timestamp": 1234567890123,
"content": [
{"type": "text", "text": "remember these attachments"},
{
"type": "image",
"upload_id": "image_1",
"name": "picture.png",
"ext": "png",
},
{
"type": "audio",
"upload_id": "audio_1",
"name": "tone.wav",
"ext": "wav",
},
],
}
]
async with app_client(config, backend) as client:
user_key = await create_user(client)
response = await client.post(
"/memories/add/multipart",
data={
"user_id": "u_123",
"user_key": user_key,
"session_id": "chat:c_uploads",
"app_id": "default",
"project_id": "default",
"messages": json.dumps(messages),
},
files=[
("image_1", ("picture.png", b"png bytes", "image/png")),
("audio_1", ("tone.wav", b"wav bytes", "audio/wav")),
],
)
assert response.status_code == 200, response.text
assert response.json() == {
"session_id": "chat:c_uploads",
"backend": {"request_id": "add", "data": {"status": "accumulated"}},
}
content = backend.add_calls[0]["messages"][0]["content"]
assert content == [
{"type": "text", "text": "remember these attachments"},
{
"type": "image",
"name": "picture.png",
"ext": "png",
"base64": base64.b64encode(b"png bytes").decode("ascii"),
},
{
"type": "audio",
"name": "tone.wav",
"ext": "wav",
"base64": base64.b64encode(b"wav bytes").decode("ascii"),
},
]
attachments = repo.list_attachments_for_session("u_123", "chat:c_uploads")
assert [(item["name"], item["source"]) for item in attachments] == [
("picture.png", "memory_add_upload"),
("tone.wav", "memory_add_upload"),
]
for attachment in attachments:
path = Path(attachment["internal_uri"].removeprefix("file://"))
assert path.exists()
@pytest.mark.asyncio
async def test_add_memory_multipart_rejects_missing_upload_file(
config: GatewayConfig,
) -> None:
backend = FakeBackendClient()
messages = [
{
"sender_id": "u_123",
"role": "user",
"timestamp": 1234567890123,
"content": [
{"type": "image", "upload_id": "image_1", "name": "picture.png"}
],
}
]
async with app_client(config, backend) as client:
user_key = await create_user(client)
response = await client.post(
"/memories/add/multipart",
data={
"user_id": "u_123",
"user_key": user_key,
"session_id": "chat:c_missing_upload",
"messages": json.dumps(messages),
},
)
assert response.status_code == 422
assert "missing upload file for upload_id: image_1" in response.text
assert backend.add_calls == []
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_add_memory_deduplicates_retried_base64_attachment( async def test_add_memory_deduplicates_retried_base64_attachment(
config: GatewayConfig, config: GatewayConfig,