tomtan
0910affc78
Some checks failed
CI / lint (push) Has been cancelled
CI / unit tests (push) Has been cancelled
CI / integration tests (push) Has been cancelled
CI / package build (push) Has been cancelled
Commit lint / pull request title (push) Has been cancelled
Commit lint / commit messages (push) Has been cancelled
441 lines
13 KiB
Python
441 lines
13 KiB
Python
"""get_llm_client — raises on missing credentials, caches on success."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import importlib
|
|
from io import BytesIO
|
|
|
|
import pytest
|
|
from everalgo.llm.types import (
|
|
ChatMessage,
|
|
ChatResponse,
|
|
ImageUrlInner,
|
|
ImageUrlPart,
|
|
TextPart,
|
|
)
|
|
from pydantic import SecretStr
|
|
|
|
from everos.component.llm import LLMNotConfiguredError
|
|
from everos.config import Settings
|
|
from everos.config.settings import LLMSettings, MultimodalSettings
|
|
|
|
_client_mod = importlib.import_module("everos.component.llm.client")
|
|
|
|
|
|
def _reset_singleton(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(_client_mod, "_llm_client", None, raising=False)
|
|
monkeypatch.setattr(_client_mod, "_multimodal_client", None, raising=False)
|
|
|
|
|
|
def _patch_settings(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
*,
|
|
api_key: str | None,
|
|
base_url: str | None,
|
|
timeout_seconds: float | None = None,
|
|
) -> None:
|
|
"""Stub the ``load_settings`` reference bound inside the client module."""
|
|
cfg = Settings(
|
|
llm=LLMSettings(
|
|
model="gpt-4o-mini",
|
|
api_key=SecretStr(api_key) if api_key is not None else None,
|
|
base_url=base_url,
|
|
**(
|
|
{"timeout_seconds": timeout_seconds}
|
|
if timeout_seconds is not None
|
|
else {}
|
|
),
|
|
)
|
|
)
|
|
monkeypatch.setattr(_client_mod, "load_settings", lambda: cfg)
|
|
|
|
|
|
def _patch_multimodal_settings(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
*,
|
|
api_key: str | None,
|
|
base_url: str | None,
|
|
timeout_seconds: float | None = None,
|
|
resize_images_for_vlm: bool | None = None,
|
|
) -> None:
|
|
cfg = Settings(
|
|
multimodal=MultimodalSettings(
|
|
model="vision-model",
|
|
api_key=SecretStr(api_key) if api_key is not None else None,
|
|
base_url=base_url,
|
|
**(
|
|
{"timeout_seconds": timeout_seconds}
|
|
if timeout_seconds is not None
|
|
else {}
|
|
),
|
|
**(
|
|
{"resize_images_for_vlm": resize_images_for_vlm}
|
|
if resize_images_for_vlm is not None
|
|
else {}
|
|
),
|
|
)
|
|
)
|
|
monkeypatch.setattr(_client_mod, "load_settings", lambda: cfg)
|
|
|
|
|
|
class _CapturingLLM:
|
|
def __init__(self) -> None:
|
|
self.messages: list[ChatMessage] | None = None
|
|
self.kwargs: dict[str, object] | None = None
|
|
|
|
async def chat(
|
|
self,
|
|
messages: list[ChatMessage],
|
|
**kwargs: object,
|
|
) -> ChatResponse:
|
|
self.messages = messages
|
|
self.kwargs = kwargs
|
|
return ChatResponse(content="ok", model="fake")
|
|
|
|
|
|
def _assert_no_thinking_param(kwargs: dict[str, object] | None) -> None:
|
|
assert kwargs is not None
|
|
extra_body = kwargs.get("extra_body")
|
|
assert isinstance(extra_body, dict)
|
|
chat_template_kwargs = extra_body.get("chat_template_kwargs")
|
|
assert isinstance(chat_template_kwargs, dict)
|
|
assert chat_template_kwargs["enable_thinking"] is False
|
|
|
|
|
|
def _png_data_url(size: tuple[int, int]) -> str:
|
|
from PIL import Image
|
|
|
|
image = Image.new("RGB", size, color=(255, 0, 0))
|
|
buffer = BytesIO()
|
|
image.save(buffer, format="PNG")
|
|
encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
|
|
return f"data:image/png;base64,{encoded}"
|
|
|
|
|
|
def _data_url_image_size(data_url: str) -> tuple[int, int]:
|
|
from PIL import Image
|
|
|
|
_, encoded = data_url.split(",", 1)
|
|
with Image.open(BytesIO(base64.b64decode(encoded))) as image:
|
|
return image.size
|
|
|
|
|
|
def test_raises_when_api_key_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_settings(monkeypatch, api_key=None, base_url="https://example.test")
|
|
|
|
with pytest.raises(LLMNotConfiguredError, match="EVEROS_LLM__API_KEY"):
|
|
_client_mod.get_llm_client()
|
|
|
|
|
|
def test_raises_when_base_url_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_settings(monkeypatch, api_key="sk-test", base_url=None)
|
|
|
|
with pytest.raises(LLMNotConfiguredError, match="EVEROS_LLM__BASE_URL"):
|
|
_client_mod.get_llm_client()
|
|
|
|
|
|
def test_returns_singleton_when_configured(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_settings(monkeypatch, api_key="sk-test", base_url="https://example.test")
|
|
sentinel = object()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: sentinel)
|
|
|
|
first = _client_mod.get_llm_client()
|
|
second = _client_mod.get_llm_client()
|
|
|
|
assert first is second
|
|
assert first._inner is sentinel
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_client_defaults_to_no_thinking_param(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_settings(monkeypatch, api_key="sk-test", base_url="https://example.test")
|
|
captured = _CapturingLLM()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured)
|
|
|
|
client = _client_mod.get_llm_client()
|
|
await client.chat([ChatMessage(role="user", content="hello")])
|
|
|
|
_assert_no_thinking_param(captured.kwargs)
|
|
|
|
|
|
def test_llm_client_passes_configured_timeout(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
timeout_seconds=180.0,
|
|
)
|
|
captured_configs = []
|
|
sentinel = object()
|
|
|
|
def capture_build_client(cfg):
|
|
captured_configs.append(cfg)
|
|
return sentinel
|
|
|
|
monkeypatch.setattr(_client_mod, "build_client", capture_build_client)
|
|
|
|
client = _client_mod.get_llm_client()
|
|
assert client._inner is sentinel
|
|
assert captured_configs[0].timeout == 180.0
|
|
|
|
|
|
def test_multimodal_client_passes_configured_timeout(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_multimodal_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
timeout_seconds=240.0,
|
|
)
|
|
captured_configs = []
|
|
sentinel = _CapturingLLM()
|
|
|
|
def capture_build_client(cfg):
|
|
captured_configs.append(cfg)
|
|
return sentinel
|
|
|
|
monkeypatch.setattr(_client_mod, "build_client", capture_build_client)
|
|
|
|
_client_mod.get_multimodal_llm_client()
|
|
assert captured_configs[0].timeout == 240.0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multimodal_client_sets_default_image_detail(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_multimodal_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
)
|
|
captured = _CapturingLLM()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured)
|
|
|
|
client = _client_mod.get_multimodal_llm_client()
|
|
original = ChatMessage(
|
|
role="user",
|
|
content=[
|
|
TextPart(text="describe"),
|
|
ImageUrlPart(
|
|
image_url=ImageUrlInner(url="data:image/jpeg;base64,abcd")
|
|
),
|
|
],
|
|
)
|
|
|
|
await client.chat([original], max_tokens=10)
|
|
|
|
assert captured.messages is not None
|
|
sent_content = captured.messages[0].content
|
|
assert isinstance(sent_content, list)
|
|
sent_image = sent_content[1]
|
|
assert isinstance(sent_image, ImageUrlPart)
|
|
assert sent_image.image_url.detail == "auto"
|
|
|
|
original_content = original.content
|
|
assert isinstance(original_content, list)
|
|
original_image = original_content[1]
|
|
assert isinstance(original_image, ImageUrlPart)
|
|
assert original_image.image_url.detail is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multimodal_client_adds_visual_memory_instructions(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_multimodal_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
)
|
|
captured = _CapturingLLM()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured)
|
|
|
|
client = _client_mod.get_multimodal_llm_client()
|
|
original = ChatMessage(
|
|
role="user",
|
|
content=[
|
|
TextPart(text="Read this image and return its content."),
|
|
ImageUrlPart(
|
|
image_url=ImageUrlInner(url="data:image/jpeg;base64,abcd")
|
|
),
|
|
],
|
|
)
|
|
|
|
await client.chat([original], max_tokens=10)
|
|
|
|
assert captured.messages is not None
|
|
sent_content = captured.messages[0].content
|
|
assert isinstance(sent_content, list)
|
|
sent_text = sent_content[0]
|
|
assert isinstance(sent_text, TextPart)
|
|
sent_text_lower = sent_text.text.lower()
|
|
assert "spatial relationships" in sent_text_lower
|
|
assert "relative positions" in sent_text_lower
|
|
assert "Do NOT describe the parser, assistant, or ChatGPT" in sent_text.text
|
|
|
|
original_content = original.content
|
|
assert isinstance(original_content, list)
|
|
original_text = original_content[0]
|
|
assert isinstance(original_text, TextPart)
|
|
assert "spatial relationships" not in original_text.text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multimodal_client_defaults_to_no_thinking_param(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_multimodal_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
)
|
|
captured = _CapturingLLM()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured)
|
|
|
|
client = _client_mod.get_multimodal_llm_client()
|
|
original = ChatMessage(
|
|
role="user",
|
|
content=[
|
|
TextPart(text="Read this image and return its content."),
|
|
ImageUrlPart(
|
|
image_url=ImageUrlInner(url="data:image/jpeg;base64,abcd")
|
|
),
|
|
],
|
|
)
|
|
|
|
await client.chat(
|
|
[original],
|
|
max_tokens=10,
|
|
extra_body={"provider": {"only": ["test"]}},
|
|
)
|
|
|
|
_assert_no_thinking_param(captured.kwargs)
|
|
assert captured.kwargs is not None
|
|
extra_body = captured.kwargs["extra_body"]
|
|
assert isinstance(extra_body, dict)
|
|
assert extra_body["provider"] == {"only": ["test"]}
|
|
assert captured.messages is not None
|
|
sent_content = captured.messages[0].content
|
|
assert isinstance(sent_content, list)
|
|
sent_text = sent_content[0]
|
|
assert isinstance(sent_text, TextPart)
|
|
assert "/no_think" not in sent_text.text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multimodal_client_resizes_landscape_image_to_64_min_side_by_default(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_multimodal_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
)
|
|
captured = _CapturingLLM()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured)
|
|
image_url = _png_data_url((640, 480))
|
|
|
|
client = _client_mod.get_multimodal_llm_client()
|
|
original = ChatMessage(
|
|
role="user",
|
|
content=[
|
|
TextPart(text="describe"),
|
|
ImageUrlPart(image_url=ImageUrlInner(url=image_url)),
|
|
],
|
|
)
|
|
|
|
await client.chat([original], max_tokens=10)
|
|
|
|
assert captured.messages is not None
|
|
sent_content = captured.messages[0].content
|
|
assert isinstance(sent_content, list)
|
|
sent_image = sent_content[1]
|
|
assert isinstance(sent_image, ImageUrlPart)
|
|
assert _data_url_image_size(sent_image.image_url.url) == (85, 64)
|
|
assert _data_url_image_size(image_url) == (640, 480)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multimodal_client_resizes_portrait_image_to_64_min_side_by_default(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_multimodal_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
)
|
|
captured = _CapturingLLM()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured)
|
|
image_url = _png_data_url((480, 640))
|
|
|
|
client = _client_mod.get_multimodal_llm_client()
|
|
original = ChatMessage(
|
|
role="user",
|
|
content=[
|
|
TextPart(text="describe"),
|
|
ImageUrlPart(image_url=ImageUrlInner(url=image_url)),
|
|
],
|
|
)
|
|
|
|
await client.chat([original], max_tokens=10)
|
|
|
|
assert captured.messages is not None
|
|
sent_content = captured.messages[0].content
|
|
assert isinstance(sent_content, list)
|
|
sent_image = sent_content[1]
|
|
assert isinstance(sent_image, ImageUrlPart)
|
|
assert _data_url_image_size(sent_image.image_url.url) == (64, 85)
|
|
assert _data_url_image_size(image_url) == (480, 640)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multimodal_client_keeps_image_when_resize_disabled(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
_reset_singleton(monkeypatch)
|
|
_patch_multimodal_settings(
|
|
monkeypatch,
|
|
api_key="sk-test",
|
|
base_url="https://example.test",
|
|
resize_images_for_vlm=False,
|
|
)
|
|
captured = _CapturingLLM()
|
|
monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured)
|
|
image_url = _png_data_url((640, 480))
|
|
|
|
client = _client_mod.get_multimodal_llm_client()
|
|
original = ChatMessage(
|
|
role="user",
|
|
content=[
|
|
TextPart(text="describe"),
|
|
ImageUrlPart(image_url=ImageUrlInner(url=image_url)),
|
|
],
|
|
)
|
|
|
|
await client.chat([original], max_tokens=10)
|
|
|
|
assert captured.messages is not None
|
|
sent_content = captured.messages[0].content
|
|
assert isinstance(sent_content, list)
|
|
sent_image = sent_content[1]
|
|
assert isinstance(sent_image, ImageUrlPart)
|
|
assert sent_image.image_url.url == image_url
|