Add persisted LLM audit logging

app-instance/backend/nanobot/llm_audit.py

@@ -0,0 +1,186 @@
+"""Structured LLM audit logging persisted in backend storage."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from nanobot.utils.helpers import get_logs_path
+
+_MAX_TEXT_PREVIEW = 1000
+_MAX_TRACEBACK_PREVIEW = 8000
+_REDACTED = "***REDACTED***"
+_SENSITIVE_KEYS = {
+    "api_key",
+    "authorization",
+    "proxy_authorization",
+    "x_api_key",
+    "x-api-key",
+    "token",
+    "access_token",
+    "refresh_token",
+    "secret",
+    "password",
+}
+
+
+def get_llm_audit_log_path() -> Path:
+    """Return the persisted LLM audit log path."""
+    return get_logs_path() / "llm_audit.jsonl"
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _truncate_text(text: str, limit: int = _MAX_TEXT_PREVIEW) -> str:
+    if len(text) <= limit:
+        return text
+    return text[:limit] + "...(truncated)"
+
+
+def _redact_value(key: str, value: Any) -> Any:
+    if key.lower() in _SENSITIVE_KEYS and value is not None:
+        return _REDACTED
+    return value
+
+
+def redact_mapping(mapping: dict[str, Any] | None) -> dict[str, Any]:
+    """Redact common secret-like keys in a mapping."""
+    if not mapping:
+        return {}
+    sanitized: dict[str, Any] = {}
+    for key, value in mapping.items():
+        if isinstance(value, dict):
+            sanitized[key] = redact_mapping(value)
+            continue
+        if isinstance(value, list):
+            sanitized[key] = [
+                redact_mapping(item) if isinstance(item, dict) else item
+                for item in value
+            ]
+            continue
+        sanitized[key] = _redact_value(str(key), value)
+    return sanitized
+
+
+def summarize_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Build a compact audit-safe summary of prompt messages."""
+    summary: list[dict[str, Any]] = []
+    for idx, msg in enumerate(messages):
+        item: dict[str, Any] = {
+            "index": idx,
+            "role": msg.get("role"),
+        }
+        if "name" in msg:
+            item["name"] = msg.get("name")
+        if "tool_call_id" in msg:
+            item["tool_call_id"] = msg.get("tool_call_id")
+
+        content = msg.get("content")
+        if content is None:
+            item["content_kind"] = "none"
+        elif isinstance(content, str):
+            item["content_kind"] = "text"
+            item["content_length"] = len(content)
+            item["content_preview"] = _truncate_text(content)
+        elif isinstance(content, list):
+            item["content_kind"] = "blocks"
+            item["content_blocks"] = len(content)
+            item["content_preview"] = _truncate_text(json.dumps(content, ensure_ascii=False))
+        else:
+            rendered = str(content)
+            item["content_kind"] = type(content).__name__
+            item["content_length"] = len(rendered)
+            item["content_preview"] = _truncate_text(rendered)
+
+        tool_calls = msg.get("tool_calls")
+        if isinstance(tool_calls, list) and tool_calls:
+            item["tool_calls"] = summarize_tool_calls(tool_calls)
+
+        summary.append(item)
+    return summary
+
+
+def summarize_tool_calls(tool_calls: list[Any]) -> list[dict[str, Any]]:
+    """Summarize outgoing or incoming tool calls."""
+    summary: list[dict[str, Any]] = []
+    for idx, tool_call in enumerate(tool_calls):
+        if hasattr(tool_call, "function"):
+            function = getattr(tool_call, "function")
+            arguments = getattr(function, "arguments", None)
+            summary.append({
+                "index": idx,
+                "id": getattr(tool_call, "id", None),
+                "name": getattr(function, "name", None),
+                "arguments_preview": _truncate_text(str(arguments) if arguments is not None else ""),
+            })
+            continue
+
+        if isinstance(tool_call, dict):
+            fn = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {}
+            summary.append({
+                "index": idx,
+                "id": tool_call.get("id"),
+                "name": fn.get("name"),
+                "arguments_preview": _truncate_text(str(fn.get("arguments", ""))),
+            })
+            continue
+
+        summary.append({
+            "index": idx,
+            "repr": _truncate_text(str(tool_call)),
+        })
+    return summary
+
+
+def summarize_tools(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
+    """Summarize tool definitions sent to the provider."""
+    if not tools:
+        return []
+    summary: list[dict[str, Any]] = []
+    for idx, tool in enumerate(tools):
+        function = tool.get("function") if isinstance(tool, dict) else None
+        entry = {
+            "index": idx,
+            "type": tool.get("type") if isinstance(tool, dict) else None,
+        }
+        if isinstance(function, dict):
+            entry["name"] = function.get("name")
+            params = function.get("parameters")
+            if params is not None:
+                entry["parameters_preview"] = _truncate_text(json.dumps(params, ensure_ascii=False))
+        else:
+            entry["preview"] = _truncate_text(str(tool))
+        summary.append(entry)
+    return summary
+
+
+def write_llm_audit_event(event: dict[str, Any]) -> None:
+    """Append one JSONL audit event to backend storage."""
+    payload = {
+        "ts": _utc_now_iso(),
+        **event,
+    }
+    path = get_llm_audit_log_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
+    except Exception as exc:
+        logger.warning("Failed to persist LLM audit log: {}", exc)
+
+
+def summarize_exception(exc: BaseException) -> dict[str, str]:
+    return {
+        "type": type(exc).__name__,
+        "message": str(exc),
+    }
+
+
+def truncate_traceback(text: str) -> str:
+    return _truncate_text(text, _MAX_TRACEBACK_PREVIEW)

app-instance/backend/nanobot/providers/litellm_provider.py

@@ -3,11 +3,23 @@
 import json
 import json_repair
 import os
+import traceback
+import uuid
 from typing import Any
 
 import litellm
 from litellm import acompletion
+from loguru import logger
 
+from nanobot.llm_audit import (
+    redact_mapping,
+    summarize_exception,
+    summarize_messages,
+    summarize_tool_calls,
+    summarize_tools,
+    truncate_traceback,
+    write_llm_audit_event,
+)
 from nanobot.providers.base import LLMProvider, LLMResponse, ToolCallRequest
 from nanobot.providers.registry import find_by_model, find_gateway
 
@@ -186,9 +198,12 @@ class LiteLLMProvider(LLMProvider):
         """
         original_model = model or self.default_model
         model = self._resolve_model(original_model)
+        request_id = str(uuid.uuid4())
+        sanitized_messages = self._sanitize_messages(self._sanitize_empty_content(messages))
 
         if self._supports_cache_control(original_model):
             messages, tools = self._apply_cache_control(messages, tools)
+            sanitized_messages = self._sanitize_messages(self._sanitize_empty_content(messages))
 
         # Clamp max_tokens to at least 1 — negative or zero values cause
         # LiteLLM to reject the request with "max_tokens must be at least 1".
@@ -196,7 +211,7 @@ class LiteLLMProvider(LLMProvider):
         
         kwargs: dict[str, Any] = {
             "model": model,
-            "messages": self._sanitize_messages(self._sanitize_empty_content(messages)),
+            "messages": sanitized_messages,
             "max_tokens": max_tokens,
             "temperature": temperature,
         }
@@ -219,11 +234,83 @@ class LiteLLMProvider(LLMProvider):
         if tools:
             kwargs["tools"] = tools
             kwargs["tool_choice"] = "auto"
+
+        request_event = {
+            "event": "llm_request",
+            "request_id": request_id,
+            "provider_impl": type(self).__name__,
+            "gateway": self._gateway.name if self._gateway else None,
+            "original_model": original_model,
+            "resolved_model": model,
+            "api_base": self.api_base,
+            "has_api_key": bool(self.api_key),
+            "temperature": kwargs.get("temperature"),
+            "max_tokens": kwargs.get("max_tokens"),
+            "tool_choice": kwargs.get("tool_choice"),
+            "message_count": len(sanitized_messages),
+            "messages": summarize_messages(sanitized_messages),
+            "tools": summarize_tools(tools),
+            "extra_headers": redact_mapping(self.extra_headers),
+        }
+        write_llm_audit_event(request_event)
+        logger.info(
+            "LLM request [{}]: model={} messages={} tools={}",
+            request_id,
+            model,
+            len(sanitized_messages),
+            len(tools or []),
+        )
         
         try:
             response = await acompletion(**kwargs)
-            return self._parse_response(response)
+            parsed = self._parse_response(response)
+            write_llm_audit_event({
+                "event": "llm_response",
+                "request_id": request_id,
+                "provider_impl": type(self).__name__,
+                "original_model": original_model,
+                "resolved_model": model,
+                "finish_reason": parsed.finish_reason,
+                "usage": parsed.usage,
+                "content_preview": parsed.content[:1000] if parsed.content else None,
+                "reasoning_preview": parsed.reasoning_content[:1000] if parsed.reasoning_content else None,
+                "tool_calls": [
+                    {
+                        "id": tc.id,
+                        "name": tc.name,
+                        "arguments_preview": str(tc.arguments)[:1000],
+                    }
+                    for tc in parsed.tool_calls
+                ],
+                "raw_tool_calls": summarize_tool_calls(
+                    getattr(response.choices[0].message, "tool_calls", None) or []
+                ),
+            })
+            logger.info(
+                "LLM response [{}]: model={} finish_reason={} tool_calls={}",
+                request_id,
+                model,
+                parsed.finish_reason,
+                len(parsed.tool_calls),
+            )
+            return parsed
         except Exception as e:
+            tb = traceback.format_exc()
+            write_llm_audit_event({
+                "event": "llm_error",
+                "request_id": request_id,
+                "provider_impl": type(self).__name__,
+                "gateway": self._gateway.name if self._gateway else None,
+                "original_model": original_model,
+                "resolved_model": model,
+                "api_base": self.api_base,
+                "error": summarize_exception(e),
+                "traceback": truncate_traceback(tb),
+                "message_count": len(sanitized_messages),
+                "messages": summarize_messages(sanitized_messages),
+                "tools": summarize_tools(tools),
+            })
+            logger.exception("LLM error [{}]: model={} provider call failed", request_id, model)
             # Return error as content for graceful handling
             return LLMResponse(
                 content=f"Error calling LLM: {str(e)}",

app-instance/backend/nanobot/utils/__init__.py

@@ -4,6 +4,7 @@ from nanobot.utils.helpers import (
     ensure_dir,
     get_cron_store_path,
     get_data_path,
+    get_logs_path,
     get_workspace_path,
     get_workspace_state_path,
 )
@@ -13,5 +14,6 @@ __all__ = [
     "get_workspace_path",
     "get_workspace_state_path",
     "get_data_path",
+    "get_logs_path",
     "get_cron_store_path",
 ]

app-instance/backend/nanobot/utils/helpers.py

@@ -42,6 +42,11 @@ def get_data_path() -> Path:
     return ensure_dir(Path.home() / ".nanobot")
 
 
+def get_logs_path() -> Path:
+    """获取后端日志目录（~/.nanobot/logs）。"""
+    return ensure_dir(get_data_path() / "logs")
+
+
 def get_legacy_cron_store_path() -> Path:
     """获取旧版全局 cron store 路径（~/.nanobot/cron/jobs.json）。"""
     return get_data_path() / "cron" / "jobs.json"