steven_li
8a12c30141
新增内部Task系统，包括验证、反馈门控机制，实现自动质量验证 (通过率>=0.75)和用户反馈闭环(satisfied/revise/abandon)。 实现Agent Team v1协调器，支持sequence/parallel/dag执行策略， sub-agent复用主AgentLoop，每个run使用独立memory snapshot。 建立Skill学习pipeline，包含draft/审核/发布/回滚完整生命周期， 通过Task验证通过且用户满意才生成学习候选。 重构目录结构，移除third_party依赖，建立统一engine内核， 所有agent共享运行时基础组件。 更新ContextBuilder清理provider消息字段，增强SkillContext版本管理， 集成TaskExecutionPlanner和TaskSkillResolver实现技能解析机制。
139 lines
5.6 KiB
Python
139 lines
5.6 KiB
Python
"""Automatic validation for internal Task mode."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from beaver.engine.providers import ProviderBundle
|
|
|
|
from .models import TaskRecord, ValidationResult
|
|
|
|
|
|
class ValidationService:
|
|
async def validate_task_result(
|
|
self,
|
|
*,
|
|
task: TaskRecord,
|
|
user_message: str,
|
|
final_output: str,
|
|
transcript_excerpt: str = "",
|
|
tool_summaries: list[str] | None = None,
|
|
team_summaries: list[str] | None = None,
|
|
provider_bundle: ProviderBundle | None = None,
|
|
) -> ValidationResult:
|
|
provider = None
|
|
model = None
|
|
if provider_bundle is not None:
|
|
provider = provider_bundle.auxiliary_provider or provider_bundle.main_provider
|
|
runtime = provider_bundle.auxiliary_runtime or provider_bundle.main_runtime
|
|
model = getattr(runtime, "model", None)
|
|
if provider is not None:
|
|
try:
|
|
return await self._validate_with_provider(
|
|
provider=provider,
|
|
model=model,
|
|
task=task,
|
|
user_message=user_message,
|
|
final_output=final_output,
|
|
transcript_excerpt=transcript_excerpt,
|
|
tool_summaries=tool_summaries or [],
|
|
team_summaries=team_summaries or [],
|
|
)
|
|
except Exception as exc:
|
|
return ValidationResult(
|
|
passed=False,
|
|
score=0.0,
|
|
issues=[f"Validator failed: {exc}"],
|
|
missing_requirements=["A valid automatic validation result is required before accepting the task."],
|
|
recommended_revision_prompt=(
|
|
"Review the task result again because automatic validation failed, "
|
|
"then provide a corrected final answer that explicitly satisfies the task goal."
|
|
),
|
|
validator="llm_error",
|
|
)
|
|
return self._heuristic_validate(final_output)
|
|
|
|
async def _validate_with_provider(
|
|
self,
|
|
*,
|
|
provider: Any,
|
|
model: str | None,
|
|
task: TaskRecord,
|
|
user_message: str,
|
|
final_output: str,
|
|
transcript_excerpt: str,
|
|
tool_summaries: list[str],
|
|
team_summaries: list[str],
|
|
) -> ValidationResult:
|
|
prompt = (
|
|
"Validate whether the assistant output satisfies the task. "
|
|
"Return only compact JSON with keys: passed, score, issues, "
|
|
"missing_requirements, recommended_revision_prompt.\n\n"
|
|
f"Task goal:\n{task.goal}\n\n"
|
|
f"Current user request:\n{user_message}\n\n"
|
|
f"Transcript excerpt:\n{transcript_excerpt[:2500]}\n\n"
|
|
f"Tool summaries:\n{json.dumps(tool_summaries[:12], ensure_ascii=False)}\n\n"
|
|
f"Team summaries:\n{json.dumps(team_summaries[:12], ensure_ascii=False)}\n\n"
|
|
f"Assistant final output:\n{final_output[:4000]}"
|
|
)
|
|
response = await provider.chat(
|
|
messages=[
|
|
{"role": "system", "content": "You are a strict task result validator."},
|
|
{"role": "user", "content": prompt},
|
|
],
|
|
tools=None,
|
|
model=model,
|
|
max_tokens=800,
|
|
temperature=0.0,
|
|
)
|
|
payload = self._parse_json_object(response.content or "")
|
|
return ValidationResult(
|
|
passed=bool(payload.get("passed")),
|
|
score=max(0.0, min(1.0, float(payload.get("score", 0.0) or 0.0))),
|
|
issues=[str(item) for item in payload.get("issues") or []],
|
|
missing_requirements=[str(item) for item in payload.get("missing_requirements") or []],
|
|
recommended_revision_prompt=str(payload.get("recommended_revision_prompt") or ""),
|
|
validator="llm",
|
|
)
|
|
|
|
@staticmethod
|
|
def _heuristic_validate(final_output: str) -> ValidationResult:
|
|
text = final_output.strip()
|
|
if not text:
|
|
return ValidationResult(
|
|
passed=False,
|
|
score=0.0,
|
|
issues=["Assistant output is empty."],
|
|
missing_requirements=["A non-empty result is required."],
|
|
recommended_revision_prompt="Produce a complete, non-empty answer for the task.",
|
|
validator="heuristic",
|
|
)
|
|
lowered = text.lower()
|
|
if "run failed before completion" in lowered or "tool loop stopped" in lowered:
|
|
return ValidationResult(
|
|
passed=False,
|
|
score=0.35,
|
|
issues=["The run did not complete cleanly."],
|
|
missing_requirements=["A successful final result is required."],
|
|
recommended_revision_prompt="Retry the task and address the failure before returning the final answer.",
|
|
validator="heuristic",
|
|
)
|
|
return ValidationResult(passed=True, score=0.85, validator="heuristic")
|
|
|
|
@staticmethod
|
|
def _parse_json_object(text: str) -> dict[str, Any]:
|
|
cleaned = text.strip()
|
|
if cleaned.startswith("```"):
|
|
cleaned = cleaned.strip("`")
|
|
if cleaned.lower().startswith("json"):
|
|
cleaned = cleaned[4:].strip()
|
|
start = cleaned.find("{")
|
|
end = cleaned.rfind("}")
|
|
if start >= 0 and end >= start:
|
|
cleaned = cleaned[start : end + 1]
|
|
payload = json.loads(cleaned)
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("validator response must be a JSON object")
|
|
return payload
|