From bad1e16ab4ee6b3f211141a8c479bc69f26b587d Mon Sep 17 00:00:00 2001 From: steven_li Date: Mon, 23 Mar 2026 11:41:43 +0800 Subject: [PATCH] Add persisted LLM audit logging --- app-instance/backend/nanobot/llm_audit.py | 186 ++++++++++++++++++ .../nanobot/providers/litellm_provider.py | 91 ++++++++- .../backend/nanobot/utils/__init__.py | 2 + app-instance/backend/nanobot/utils/helpers.py | 5 + 4 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 app-instance/backend/nanobot/llm_audit.py diff --git a/app-instance/backend/nanobot/llm_audit.py b/app-instance/backend/nanobot/llm_audit.py new file mode 100644 index 0000000..44b8c9c --- /dev/null +++ b/app-instance/backend/nanobot/llm_audit.py @@ -0,0 +1,186 @@ +"""Structured LLM audit logging persisted in backend storage.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from loguru import logger + +from nanobot.utils.helpers import get_logs_path + +_MAX_TEXT_PREVIEW = 1000 +_MAX_TRACEBACK_PREVIEW = 8000 +_REDACTED = "***REDACTED***" +_SENSITIVE_KEYS = { + "api_key", + "authorization", + "proxy_authorization", + "x_api_key", + "x-api-key", + "token", + "access_token", + "refresh_token", + "secret", + "password", +} + + +def get_llm_audit_log_path() -> Path: + """Return the persisted LLM audit log path.""" + return get_logs_path() / "llm_audit.jsonl" + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _truncate_text(text: str, limit: int = _MAX_TEXT_PREVIEW) -> str: + if len(text) <= limit: + return text + return text[:limit] + "...(truncated)" + + +def _redact_value(key: str, value: Any) -> Any: + if key.lower() in _SENSITIVE_KEYS and value is not None: + return _REDACTED + return value + + +def redact_mapping(mapping: dict[str, Any] | None) -> dict[str, Any]: + """Redact common secret-like keys in a mapping.""" + if not mapping: + return {} + sanitized: dict[str, Any] = {} + for key, value in mapping.items(): + if isinstance(value, dict): + sanitized[key] = redact_mapping(value) + continue + if isinstance(value, list): + sanitized[key] = [ + redact_mapping(item) if isinstance(item, dict) else item + for item in value + ] + continue + sanitized[key] = _redact_value(str(key), value) + return sanitized + + +def summarize_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Build a compact audit-safe summary of prompt messages.""" + summary: list[dict[str, Any]] = [] + for idx, msg in enumerate(messages): + item: dict[str, Any] = { + "index": idx, + "role": msg.get("role"), + } + if "name" in msg: + item["name"] = msg.get("name") + if "tool_call_id" in msg: + item["tool_call_id"] = msg.get("tool_call_id") + + content = msg.get("content") + if content is None: + item["content_kind"] = "none" + elif isinstance(content, str): + item["content_kind"] = "text" + item["content_length"] = len(content) + item["content_preview"] = _truncate_text(content) + elif isinstance(content, list): + item["content_kind"] = "blocks" + item["content_blocks"] = len(content) + item["content_preview"] = _truncate_text(json.dumps(content, ensure_ascii=False)) + else: + rendered = str(content) + item["content_kind"] = type(content).__name__ + item["content_length"] = len(rendered) + item["content_preview"] = _truncate_text(rendered) + + tool_calls = msg.get("tool_calls") + if isinstance(tool_calls, list) and tool_calls: + item["tool_calls"] = summarize_tool_calls(tool_calls) + + summary.append(item) + return summary + + +def summarize_tool_calls(tool_calls: list[Any]) -> list[dict[str, Any]]: + """Summarize outgoing or incoming tool calls.""" + summary: list[dict[str, Any]] = [] + for idx, tool_call in enumerate(tool_calls): + if hasattr(tool_call, "function"): + function = getattr(tool_call, "function") + arguments = getattr(function, "arguments", None) + summary.append({ + "index": idx, + "id": getattr(tool_call, "id", None), + "name": getattr(function, "name", None), + "arguments_preview": _truncate_text(str(arguments) if arguments is not None else ""), + }) + continue + + if isinstance(tool_call, dict): + fn = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {} + summary.append({ + "index": idx, + "id": tool_call.get("id"), + "name": fn.get("name"), + "arguments_preview": _truncate_text(str(fn.get("arguments", ""))), + }) + continue + + summary.append({ + "index": idx, + "repr": _truncate_text(str(tool_call)), + }) + return summary + + +def summarize_tools(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + """Summarize tool definitions sent to the provider.""" + if not tools: + return [] + summary: list[dict[str, Any]] = [] + for idx, tool in enumerate(tools): + function = tool.get("function") if isinstance(tool, dict) else None + entry = { + "index": idx, + "type": tool.get("type") if isinstance(tool, dict) else None, + } + if isinstance(function, dict): + entry["name"] = function.get("name") + params = function.get("parameters") + if params is not None: + entry["parameters_preview"] = _truncate_text(json.dumps(params, ensure_ascii=False)) + else: + entry["preview"] = _truncate_text(str(tool)) + summary.append(entry) + return summary + + +def write_llm_audit_event(event: dict[str, Any]) -> None: + """Append one JSONL audit event to backend storage.""" + payload = { + "ts": _utc_now_iso(), + **event, + } + path = get_llm_audit_log_path() + path.parent.mkdir(parents=True, exist_ok=True) + try: + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(payload, ensure_ascii=False) + "\n") + except Exception as exc: + logger.warning("Failed to persist LLM audit log: {}", exc) + + +def summarize_exception(exc: BaseException) -> dict[str, str]: + return { + "type": type(exc).__name__, + "message": str(exc), + } + + +def truncate_traceback(text: str) -> str: + return _truncate_text(text, _MAX_TRACEBACK_PREVIEW) diff --git a/app-instance/backend/nanobot/providers/litellm_provider.py b/app-instance/backend/nanobot/providers/litellm_provider.py index 7402a2b..29ecb6f 100644 --- a/app-instance/backend/nanobot/providers/litellm_provider.py +++ b/app-instance/backend/nanobot/providers/litellm_provider.py @@ -3,11 +3,23 @@ import json import json_repair import os +import traceback +import uuid from typing import Any import litellm from litellm import acompletion +from loguru import logger +from nanobot.llm_audit import ( + redact_mapping, + summarize_exception, + summarize_messages, + summarize_tool_calls, + summarize_tools, + truncate_traceback, + write_llm_audit_event, +) from nanobot.providers.base import LLMProvider, LLMResponse, ToolCallRequest from nanobot.providers.registry import find_by_model, find_gateway @@ -186,9 +198,12 @@ class LiteLLMProvider(LLMProvider): """ original_model = model or self.default_model model = self._resolve_model(original_model) + request_id = str(uuid.uuid4()) + sanitized_messages = self._sanitize_messages(self._sanitize_empty_content(messages)) if self._supports_cache_control(original_model): messages, tools = self._apply_cache_control(messages, tools) + sanitized_messages = self._sanitize_messages(self._sanitize_empty_content(messages)) # Clamp max_tokens to at least 1 — negative or zero values cause # LiteLLM to reject the request with "max_tokens must be at least 1". @@ -196,7 +211,7 @@ class LiteLLMProvider(LLMProvider): kwargs: dict[str, Any] = { "model": model, - "messages": self._sanitize_messages(self._sanitize_empty_content(messages)), + "messages": sanitized_messages, "max_tokens": max_tokens, "temperature": temperature, } @@ -219,11 +234,83 @@ class LiteLLMProvider(LLMProvider): if tools: kwargs["tools"] = tools kwargs["tool_choice"] = "auto" + + request_event = { + "event": "llm_request", + "request_id": request_id, + "provider_impl": type(self).__name__, + "gateway": self._gateway.name if self._gateway else None, + "original_model": original_model, + "resolved_model": model, + "api_base": self.api_base, + "has_api_key": bool(self.api_key), + "temperature": kwargs.get("temperature"), + "max_tokens": kwargs.get("max_tokens"), + "tool_choice": kwargs.get("tool_choice"), + "message_count": len(sanitized_messages), + "messages": summarize_messages(sanitized_messages), + "tools": summarize_tools(tools), + "extra_headers": redact_mapping(self.extra_headers), + } + write_llm_audit_event(request_event) + logger.info( + "LLM request [{}]: model={} messages={} tools={}", + request_id, + model, + len(sanitized_messages), + len(tools or []), + ) try: response = await acompletion(**kwargs) - return self._parse_response(response) + parsed = self._parse_response(response) + write_llm_audit_event({ + "event": "llm_response", + "request_id": request_id, + "provider_impl": type(self).__name__, + "original_model": original_model, + "resolved_model": model, + "finish_reason": parsed.finish_reason, + "usage": parsed.usage, + "content_preview": parsed.content[:1000] if parsed.content else None, + "reasoning_preview": parsed.reasoning_content[:1000] if parsed.reasoning_content else None, + "tool_calls": [ + { + "id": tc.id, + "name": tc.name, + "arguments_preview": str(tc.arguments)[:1000], + } + for tc in parsed.tool_calls + ], + "raw_tool_calls": summarize_tool_calls( + getattr(response.choices[0].message, "tool_calls", None) or [] + ), + }) + logger.info( + "LLM response [{}]: model={} finish_reason={} tool_calls={}", + request_id, + model, + parsed.finish_reason, + len(parsed.tool_calls), + ) + return parsed except Exception as e: + tb = traceback.format_exc() + write_llm_audit_event({ + "event": "llm_error", + "request_id": request_id, + "provider_impl": type(self).__name__, + "gateway": self._gateway.name if self._gateway else None, + "original_model": original_model, + "resolved_model": model, + "api_base": self.api_base, + "error": summarize_exception(e), + "traceback": truncate_traceback(tb), + "message_count": len(sanitized_messages), + "messages": summarize_messages(sanitized_messages), + "tools": summarize_tools(tools), + }) + logger.exception("LLM error [{}]: model={} provider call failed", request_id, model) # Return error as content for graceful handling return LLMResponse( content=f"Error calling LLM: {str(e)}", diff --git a/app-instance/backend/nanobot/utils/__init__.py b/app-instance/backend/nanobot/utils/__init__.py index 80ff7dd..e65576b 100644 --- a/app-instance/backend/nanobot/utils/__init__.py +++ b/app-instance/backend/nanobot/utils/__init__.py @@ -4,6 +4,7 @@ from nanobot.utils.helpers import ( ensure_dir, get_cron_store_path, get_data_path, + get_logs_path, get_workspace_path, get_workspace_state_path, ) @@ -13,5 +14,6 @@ __all__ = [ "get_workspace_path", "get_workspace_state_path", "get_data_path", + "get_logs_path", "get_cron_store_path", ] diff --git a/app-instance/backend/nanobot/utils/helpers.py b/app-instance/backend/nanobot/utils/helpers.py index b38d00b..ce55635 100644 --- a/app-instance/backend/nanobot/utils/helpers.py +++ b/app-instance/backend/nanobot/utils/helpers.py @@ -42,6 +42,11 @@ def get_data_path() -> Path: return ensure_dir(Path.home() / ".nanobot") +def get_logs_path() -> Path: + """获取后端日志目录（~/.nanobot/logs）。""" + return ensure_dir(get_data_path() / "logs") + + def get_legacy_cron_store_path() -> Path: """获取旧版全局 cron store 路径（~/.nanobot/cron/jobs.json）。""" return get_data_path() / "cron" / "jobs.json"