"""get_llm_client — raises on missing credentials, caches on success.""" from __future__ import annotations import base64 import importlib from io import BytesIO import pytest from everalgo.llm.types import ( ChatMessage, ChatResponse, ImageUrlInner, ImageUrlPart, TextPart, ) from pydantic import SecretStr from everos.component.llm import LLMNotConfiguredError from everos.config import Settings from everos.config.settings import LLMSettings, MultimodalSettings _client_mod = importlib.import_module("everos.component.llm.client") def _reset_singleton(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(_client_mod, "_llm_client", None, raising=False) monkeypatch.setattr(_client_mod, "_multimodal_client", None, raising=False) def _patch_settings( monkeypatch: pytest.MonkeyPatch, *, api_key: str | None, base_url: str | None, timeout_seconds: float | None = None, ) -> None: """Stub the ``load_settings`` reference bound inside the client module.""" cfg = Settings( llm=LLMSettings( model="gpt-4o-mini", api_key=SecretStr(api_key) if api_key is not None else None, base_url=base_url, **( {"timeout_seconds": timeout_seconds} if timeout_seconds is not None else {} ), ) ) monkeypatch.setattr(_client_mod, "load_settings", lambda: cfg) def _patch_multimodal_settings( monkeypatch: pytest.MonkeyPatch, *, api_key: str | None, base_url: str | None, timeout_seconds: float | None = None, resize_images_for_vlm: bool | None = None, ) -> None: cfg = Settings( multimodal=MultimodalSettings( model="vision-model", api_key=SecretStr(api_key) if api_key is not None else None, base_url=base_url, **( {"timeout_seconds": timeout_seconds} if timeout_seconds is not None else {} ), **( {"resize_images_for_vlm": resize_images_for_vlm} if resize_images_for_vlm is not None else {} ), ) ) monkeypatch.setattr(_client_mod, "load_settings", lambda: cfg) class _CapturingLLM: def __init__(self) -> None: self.messages: list[ChatMessage] | None = None self.kwargs: dict[str, object] | None = None async def chat( self, messages: list[ChatMessage], **kwargs: object, ) -> ChatResponse: self.messages = messages self.kwargs = kwargs return ChatResponse(content="ok", model="fake") def _assert_no_thinking_param(kwargs: dict[str, object] | None) -> None: assert kwargs is not None extra_body = kwargs.get("extra_body") assert isinstance(extra_body, dict) chat_template_kwargs = extra_body.get("chat_template_kwargs") assert isinstance(chat_template_kwargs, dict) assert chat_template_kwargs["enable_thinking"] is False def _png_data_url(size: tuple[int, int]) -> str: from PIL import Image image = Image.new("RGB", size, color=(255, 0, 0)) buffer = BytesIO() image.save(buffer, format="PNG") encoded = base64.b64encode(buffer.getvalue()).decode("ascii") return f"data:image/png;base64,{encoded}" def _data_url_image_size(data_url: str) -> tuple[int, int]: from PIL import Image _, encoded = data_url.split(",", 1) with Image.open(BytesIO(base64.b64decode(encoded))) as image: return image.size def test_raises_when_api_key_missing(monkeypatch: pytest.MonkeyPatch) -> None: _reset_singleton(monkeypatch) _patch_settings(monkeypatch, api_key=None, base_url="https://example.test") with pytest.raises(LLMNotConfiguredError, match="EVEROS_LLM__API_KEY"): _client_mod.get_llm_client() def test_raises_when_base_url_missing(monkeypatch: pytest.MonkeyPatch) -> None: _reset_singleton(monkeypatch) _patch_settings(monkeypatch, api_key="sk-test", base_url=None) with pytest.raises(LLMNotConfiguredError, match="EVEROS_LLM__BASE_URL"): _client_mod.get_llm_client() def test_returns_singleton_when_configured(monkeypatch: pytest.MonkeyPatch) -> None: _reset_singleton(monkeypatch) _patch_settings(monkeypatch, api_key="sk-test", base_url="https://example.test") sentinel = object() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: sentinel) first = _client_mod.get_llm_client() second = _client_mod.get_llm_client() assert first is second assert first._inner is sentinel @pytest.mark.asyncio async def test_llm_client_defaults_to_no_thinking_param( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_settings(monkeypatch, api_key="sk-test", base_url="https://example.test") captured = _CapturingLLM() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured) client = _client_mod.get_llm_client() await client.chat([ChatMessage(role="user", content="hello")]) _assert_no_thinking_param(captured.kwargs) def test_llm_client_passes_configured_timeout( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", timeout_seconds=180.0, ) captured_configs = [] sentinel = object() def capture_build_client(cfg): captured_configs.append(cfg) return sentinel monkeypatch.setattr(_client_mod, "build_client", capture_build_client) client = _client_mod.get_llm_client() assert client._inner is sentinel assert captured_configs[0].timeout == 180.0 def test_multimodal_client_passes_configured_timeout( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_multimodal_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", timeout_seconds=240.0, ) captured_configs = [] sentinel = _CapturingLLM() def capture_build_client(cfg): captured_configs.append(cfg) return sentinel monkeypatch.setattr(_client_mod, "build_client", capture_build_client) _client_mod.get_multimodal_llm_client() assert captured_configs[0].timeout == 240.0 @pytest.mark.asyncio async def test_multimodal_client_sets_default_image_detail( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_multimodal_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", ) captured = _CapturingLLM() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured) client = _client_mod.get_multimodal_llm_client() original = ChatMessage( role="user", content=[ TextPart(text="describe"), ImageUrlPart( image_url=ImageUrlInner(url="data:image/jpeg;base64,abcd") ), ], ) await client.chat([original], max_tokens=10) assert captured.messages is not None sent_content = captured.messages[0].content assert isinstance(sent_content, list) sent_image = sent_content[1] assert isinstance(sent_image, ImageUrlPart) assert sent_image.image_url.detail == "auto" original_content = original.content assert isinstance(original_content, list) original_image = original_content[1] assert isinstance(original_image, ImageUrlPart) assert original_image.image_url.detail is None @pytest.mark.asyncio async def test_multimodal_client_adds_visual_memory_instructions( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_multimodal_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", ) captured = _CapturingLLM() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured) client = _client_mod.get_multimodal_llm_client() original = ChatMessage( role="user", content=[ TextPart(text="Read this image and return its content."), ImageUrlPart( image_url=ImageUrlInner(url="data:image/jpeg;base64,abcd") ), ], ) await client.chat([original], max_tokens=10) assert captured.messages is not None sent_content = captured.messages[0].content assert isinstance(sent_content, list) sent_text = sent_content[0] assert isinstance(sent_text, TextPart) sent_text_lower = sent_text.text.lower() assert "spatial relationships" in sent_text_lower assert "relative positions" in sent_text_lower assert "Do NOT describe the parser, assistant, or ChatGPT" in sent_text.text original_content = original.content assert isinstance(original_content, list) original_text = original_content[0] assert isinstance(original_text, TextPart) assert "spatial relationships" not in original_text.text @pytest.mark.asyncio async def test_multimodal_client_defaults_to_no_thinking_param( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_multimodal_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", ) captured = _CapturingLLM() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured) client = _client_mod.get_multimodal_llm_client() original = ChatMessage( role="user", content=[ TextPart(text="Read this image and return its content."), ImageUrlPart( image_url=ImageUrlInner(url="data:image/jpeg;base64,abcd") ), ], ) await client.chat( [original], max_tokens=10, extra_body={"provider": {"only": ["test"]}}, ) _assert_no_thinking_param(captured.kwargs) assert captured.kwargs is not None extra_body = captured.kwargs["extra_body"] assert isinstance(extra_body, dict) assert extra_body["provider"] == {"only": ["test"]} assert captured.messages is not None sent_content = captured.messages[0].content assert isinstance(sent_content, list) sent_text = sent_content[0] assert isinstance(sent_text, TextPart) assert "/no_think" not in sent_text.text @pytest.mark.asyncio async def test_multimodal_client_resizes_landscape_image_to_64_min_side_by_default( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_multimodal_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", ) captured = _CapturingLLM() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured) image_url = _png_data_url((640, 480)) client = _client_mod.get_multimodal_llm_client() original = ChatMessage( role="user", content=[ TextPart(text="describe"), ImageUrlPart(image_url=ImageUrlInner(url=image_url)), ], ) await client.chat([original], max_tokens=10) assert captured.messages is not None sent_content = captured.messages[0].content assert isinstance(sent_content, list) sent_image = sent_content[1] assert isinstance(sent_image, ImageUrlPart) assert _data_url_image_size(sent_image.image_url.url) == (85, 64) assert _data_url_image_size(image_url) == (640, 480) @pytest.mark.asyncio async def test_multimodal_client_resizes_portrait_image_to_64_min_side_by_default( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_multimodal_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", ) captured = _CapturingLLM() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured) image_url = _png_data_url((480, 640)) client = _client_mod.get_multimodal_llm_client() original = ChatMessage( role="user", content=[ TextPart(text="describe"), ImageUrlPart(image_url=ImageUrlInner(url=image_url)), ], ) await client.chat([original], max_tokens=10) assert captured.messages is not None sent_content = captured.messages[0].content assert isinstance(sent_content, list) sent_image = sent_content[1] assert isinstance(sent_image, ImageUrlPart) assert _data_url_image_size(sent_image.image_url.url) == (64, 85) assert _data_url_image_size(image_url) == (480, 640) @pytest.mark.asyncio async def test_multimodal_client_keeps_image_when_resize_disabled( monkeypatch: pytest.MonkeyPatch, ) -> None: _reset_singleton(monkeypatch) _patch_multimodal_settings( monkeypatch, api_key="sk-test", base_url="https://example.test", resize_images_for_vlm=False, ) captured = _CapturingLLM() monkeypatch.setattr(_client_mod, "build_client", lambda cfg: captured) image_url = _png_data_url((640, 480)) client = _client_mod.get_multimodal_llm_client() original = ChatMessage( role="user", content=[ TextPart(text="describe"), ImageUrlPart(image_url=ImageUrlInner(url=image_url)), ], ) await client.chat([original], max_tokens=10) assert captured.messages is not None sent_content = captured.messages[0].content assert isinstance(sent_content, list) sent_image = sent_content[1] assert isinstance(sent_image, ImageUrlPart) assert sent_image.image_url.url == image_url