Add persisted LLM audit logging
This commit is contained in:
186
app-instance/backend/nanobot/llm_audit.py
Normal file
186
app-instance/backend/nanobot/llm_audit.py
Normal file
@ -0,0 +1,186 @@
|
||||
"""Structured LLM audit logging persisted in backend storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from nanobot.utils.helpers import get_logs_path
|
||||
|
||||
_MAX_TEXT_PREVIEW = 1000
|
||||
_MAX_TRACEBACK_PREVIEW = 8000
|
||||
_REDACTED = "***REDACTED***"
|
||||
_SENSITIVE_KEYS = {
|
||||
"api_key",
|
||||
"authorization",
|
||||
"proxy_authorization",
|
||||
"x_api_key",
|
||||
"x-api-key",
|
||||
"token",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"secret",
|
||||
"password",
|
||||
}
|
||||
|
||||
|
||||
def get_llm_audit_log_path() -> Path:
|
||||
"""Return the persisted LLM audit log path."""
|
||||
return get_logs_path() / "llm_audit.jsonl"
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _truncate_text(text: str, limit: int = _MAX_TEXT_PREVIEW) -> str:
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return text[:limit] + "...(truncated)"
|
||||
|
||||
|
||||
def _redact_value(key: str, value: Any) -> Any:
|
||||
if key.lower() in _SENSITIVE_KEYS and value is not None:
|
||||
return _REDACTED
|
||||
return value
|
||||
|
||||
|
||||
def redact_mapping(mapping: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Redact common secret-like keys in a mapping."""
|
||||
if not mapping:
|
||||
return {}
|
||||
sanitized: dict[str, Any] = {}
|
||||
for key, value in mapping.items():
|
||||
if isinstance(value, dict):
|
||||
sanitized[key] = redact_mapping(value)
|
||||
continue
|
||||
if isinstance(value, list):
|
||||
sanitized[key] = [
|
||||
redact_mapping(item) if isinstance(item, dict) else item
|
||||
for item in value
|
||||
]
|
||||
continue
|
||||
sanitized[key] = _redact_value(str(key), value)
|
||||
return sanitized
|
||||
|
||||
|
||||
def summarize_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Build a compact audit-safe summary of prompt messages."""
|
||||
summary: list[dict[str, Any]] = []
|
||||
for idx, msg in enumerate(messages):
|
||||
item: dict[str, Any] = {
|
||||
"index": idx,
|
||||
"role": msg.get("role"),
|
||||
}
|
||||
if "name" in msg:
|
||||
item["name"] = msg.get("name")
|
||||
if "tool_call_id" in msg:
|
||||
item["tool_call_id"] = msg.get("tool_call_id")
|
||||
|
||||
content = msg.get("content")
|
||||
if content is None:
|
||||
item["content_kind"] = "none"
|
||||
elif isinstance(content, str):
|
||||
item["content_kind"] = "text"
|
||||
item["content_length"] = len(content)
|
||||
item["content_preview"] = _truncate_text(content)
|
||||
elif isinstance(content, list):
|
||||
item["content_kind"] = "blocks"
|
||||
item["content_blocks"] = len(content)
|
||||
item["content_preview"] = _truncate_text(json.dumps(content, ensure_ascii=False))
|
||||
else:
|
||||
rendered = str(content)
|
||||
item["content_kind"] = type(content).__name__
|
||||
item["content_length"] = len(rendered)
|
||||
item["content_preview"] = _truncate_text(rendered)
|
||||
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if isinstance(tool_calls, list) and tool_calls:
|
||||
item["tool_calls"] = summarize_tool_calls(tool_calls)
|
||||
|
||||
summary.append(item)
|
||||
return summary
|
||||
|
||||
|
||||
def summarize_tool_calls(tool_calls: list[Any]) -> list[dict[str, Any]]:
|
||||
"""Summarize outgoing or incoming tool calls."""
|
||||
summary: list[dict[str, Any]] = []
|
||||
for idx, tool_call in enumerate(tool_calls):
|
||||
if hasattr(tool_call, "function"):
|
||||
function = getattr(tool_call, "function")
|
||||
arguments = getattr(function, "arguments", None)
|
||||
summary.append({
|
||||
"index": idx,
|
||||
"id": getattr(tool_call, "id", None),
|
||||
"name": getattr(function, "name", None),
|
||||
"arguments_preview": _truncate_text(str(arguments) if arguments is not None else ""),
|
||||
})
|
||||
continue
|
||||
|
||||
if isinstance(tool_call, dict):
|
||||
fn = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {}
|
||||
summary.append({
|
||||
"index": idx,
|
||||
"id": tool_call.get("id"),
|
||||
"name": fn.get("name"),
|
||||
"arguments_preview": _truncate_text(str(fn.get("arguments", ""))),
|
||||
})
|
||||
continue
|
||||
|
||||
summary.append({
|
||||
"index": idx,
|
||||
"repr": _truncate_text(str(tool_call)),
|
||||
})
|
||||
return summary
|
||||
|
||||
|
||||
def summarize_tools(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
|
||||
"""Summarize tool definitions sent to the provider."""
|
||||
if not tools:
|
||||
return []
|
||||
summary: list[dict[str, Any]] = []
|
||||
for idx, tool in enumerate(tools):
|
||||
function = tool.get("function") if isinstance(tool, dict) else None
|
||||
entry = {
|
||||
"index": idx,
|
||||
"type": tool.get("type") if isinstance(tool, dict) else None,
|
||||
}
|
||||
if isinstance(function, dict):
|
||||
entry["name"] = function.get("name")
|
||||
params = function.get("parameters")
|
||||
if params is not None:
|
||||
entry["parameters_preview"] = _truncate_text(json.dumps(params, ensure_ascii=False))
|
||||
else:
|
||||
entry["preview"] = _truncate_text(str(tool))
|
||||
summary.append(entry)
|
||||
return summary
|
||||
|
||||
|
||||
def write_llm_audit_event(event: dict[str, Any]) -> None:
|
||||
"""Append one JSONL audit event to backend storage."""
|
||||
payload = {
|
||||
"ts": _utc_now_iso(),
|
||||
**event,
|
||||
}
|
||||
path = get_llm_audit_log_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
with path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to persist LLM audit log: {}", exc)
|
||||
|
||||
|
||||
def summarize_exception(exc: BaseException) -> dict[str, str]:
|
||||
return {
|
||||
"type": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
}
|
||||
|
||||
|
||||
def truncate_traceback(text: str) -> str:
|
||||
return _truncate_text(text, _MAX_TRACEBACK_PREVIEW)
|
||||
@ -3,11 +3,23 @@
|
||||
import json
|
||||
import json_repair
|
||||
import os
|
||||
import traceback
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from litellm import acompletion
|
||||
from loguru import logger
|
||||
|
||||
from nanobot.llm_audit import (
|
||||
redact_mapping,
|
||||
summarize_exception,
|
||||
summarize_messages,
|
||||
summarize_tool_calls,
|
||||
summarize_tools,
|
||||
truncate_traceback,
|
||||
write_llm_audit_event,
|
||||
)
|
||||
from nanobot.providers.base import LLMProvider, LLMResponse, ToolCallRequest
|
||||
from nanobot.providers.registry import find_by_model, find_gateway
|
||||
|
||||
@ -186,9 +198,12 @@ class LiteLLMProvider(LLMProvider):
|
||||
"""
|
||||
original_model = model or self.default_model
|
||||
model = self._resolve_model(original_model)
|
||||
request_id = str(uuid.uuid4())
|
||||
sanitized_messages = self._sanitize_messages(self._sanitize_empty_content(messages))
|
||||
|
||||
if self._supports_cache_control(original_model):
|
||||
messages, tools = self._apply_cache_control(messages, tools)
|
||||
sanitized_messages = self._sanitize_messages(self._sanitize_empty_content(messages))
|
||||
|
||||
# Clamp max_tokens to at least 1 — negative or zero values cause
|
||||
# LiteLLM to reject the request with "max_tokens must be at least 1".
|
||||
@ -196,7 +211,7 @@ class LiteLLMProvider(LLMProvider):
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": self._sanitize_messages(self._sanitize_empty_content(messages)),
|
||||
"messages": sanitized_messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
}
|
||||
@ -219,11 +234,83 @@ class LiteLLMProvider(LLMProvider):
|
||||
if tools:
|
||||
kwargs["tools"] = tools
|
||||
kwargs["tool_choice"] = "auto"
|
||||
|
||||
request_event = {
|
||||
"event": "llm_request",
|
||||
"request_id": request_id,
|
||||
"provider_impl": type(self).__name__,
|
||||
"gateway": self._gateway.name if self._gateway else None,
|
||||
"original_model": original_model,
|
||||
"resolved_model": model,
|
||||
"api_base": self.api_base,
|
||||
"has_api_key": bool(self.api_key),
|
||||
"temperature": kwargs.get("temperature"),
|
||||
"max_tokens": kwargs.get("max_tokens"),
|
||||
"tool_choice": kwargs.get("tool_choice"),
|
||||
"message_count": len(sanitized_messages),
|
||||
"messages": summarize_messages(sanitized_messages),
|
||||
"tools": summarize_tools(tools),
|
||||
"extra_headers": redact_mapping(self.extra_headers),
|
||||
}
|
||||
write_llm_audit_event(request_event)
|
||||
logger.info(
|
||||
"LLM request [{}]: model={} messages={} tools={}",
|
||||
request_id,
|
||||
model,
|
||||
len(sanitized_messages),
|
||||
len(tools or []),
|
||||
)
|
||||
|
||||
try:
|
||||
response = await acompletion(**kwargs)
|
||||
return self._parse_response(response)
|
||||
parsed = self._parse_response(response)
|
||||
write_llm_audit_event({
|
||||
"event": "llm_response",
|
||||
"request_id": request_id,
|
||||
"provider_impl": type(self).__name__,
|
||||
"original_model": original_model,
|
||||
"resolved_model": model,
|
||||
"finish_reason": parsed.finish_reason,
|
||||
"usage": parsed.usage,
|
||||
"content_preview": parsed.content[:1000] if parsed.content else None,
|
||||
"reasoning_preview": parsed.reasoning_content[:1000] if parsed.reasoning_content else None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": tc.id,
|
||||
"name": tc.name,
|
||||
"arguments_preview": str(tc.arguments)[:1000],
|
||||
}
|
||||
for tc in parsed.tool_calls
|
||||
],
|
||||
"raw_tool_calls": summarize_tool_calls(
|
||||
getattr(response.choices[0].message, "tool_calls", None) or []
|
||||
),
|
||||
})
|
||||
logger.info(
|
||||
"LLM response [{}]: model={} finish_reason={} tool_calls={}",
|
||||
request_id,
|
||||
model,
|
||||
parsed.finish_reason,
|
||||
len(parsed.tool_calls),
|
||||
)
|
||||
return parsed
|
||||
except Exception as e:
|
||||
tb = traceback.format_exc()
|
||||
write_llm_audit_event({
|
||||
"event": "llm_error",
|
||||
"request_id": request_id,
|
||||
"provider_impl": type(self).__name__,
|
||||
"gateway": self._gateway.name if self._gateway else None,
|
||||
"original_model": original_model,
|
||||
"resolved_model": model,
|
||||
"api_base": self.api_base,
|
||||
"error": summarize_exception(e),
|
||||
"traceback": truncate_traceback(tb),
|
||||
"message_count": len(sanitized_messages),
|
||||
"messages": summarize_messages(sanitized_messages),
|
||||
"tools": summarize_tools(tools),
|
||||
})
|
||||
logger.exception("LLM error [{}]: model={} provider call failed", request_id, model)
|
||||
# Return error as content for graceful handling
|
||||
return LLMResponse(
|
||||
content=f"Error calling LLM: {str(e)}",
|
||||
|
||||
@ -4,6 +4,7 @@ from nanobot.utils.helpers import (
|
||||
ensure_dir,
|
||||
get_cron_store_path,
|
||||
get_data_path,
|
||||
get_logs_path,
|
||||
get_workspace_path,
|
||||
get_workspace_state_path,
|
||||
)
|
||||
@ -13,5 +14,6 @@ __all__ = [
|
||||
"get_workspace_path",
|
||||
"get_workspace_state_path",
|
||||
"get_data_path",
|
||||
"get_logs_path",
|
||||
"get_cron_store_path",
|
||||
]
|
||||
|
||||
@ -42,6 +42,11 @@ def get_data_path() -> Path:
|
||||
return ensure_dir(Path.home() / ".nanobot")
|
||||
|
||||
|
||||
def get_logs_path() -> Path:
|
||||
"""获取后端日志目录（~/.nanobot/logs）。"""
|
||||
return ensure_dir(get_data_path() / "logs")
|
||||
|
||||
|
||||
def get_legacy_cron_store_path() -> Path:
|
||||
"""获取旧版全局 cron store 路径（~/.nanobot/cron/jobs.json）。"""
|
||||
return get_data_path() / "cron" / "jobs.json"
|
||||
|
||||
Reference in New Issue
Block a user