feat(skill-learning): produce replay eval reports
This commit is contained in:
@ -8,6 +8,7 @@ from beaver.engine.providers import ProviderBundle
|
||||
from beaver.memory.skills import SkillDraftEvalReport, SkillDraftSafetyReport, SkillLearningCandidate, SkillLearningStore
|
||||
from beaver.skills.drafts import DraftService
|
||||
from beaver.skills.learning.eval import SkillDraftEvaluator
|
||||
from beaver.skills.learning.replay import ReplayRunner
|
||||
from beaver.skills.learning.service import SkillLearningService
|
||||
from beaver.skills.learning.safety import SkillDraftSafetyChecker
|
||||
from beaver.skills.publisher import SkillPublisher
|
||||
@ -285,11 +286,17 @@ class SkillLearningPipelineService:
|
||||
draft_id: str,
|
||||
*,
|
||||
provider_bundle: ProviderBundle | None,
|
||||
replay_runner: ReplayRunner | None = None,
|
||||
) -> SkillDraftEvalReport:
|
||||
draft = self.get_draft(skill_name, draft_id)
|
||||
candidate = self.get_candidate(candidate_id)
|
||||
evaluator = self.evaluator or SkillDraftEvaluator(self.learning_service.run_store)
|
||||
report = await evaluator.evaluate(candidate=candidate, draft=draft, provider_bundle=provider_bundle)
|
||||
report = await evaluator.evaluate(
|
||||
candidate=candidate,
|
||||
draft=draft,
|
||||
provider_bundle=provider_bundle,
|
||||
replay_runner=replay_runner,
|
||||
)
|
||||
self.learning_store.write_eval_report(report)
|
||||
if report.status == "skipped_provider_unavailable":
|
||||
status = "draft_ready"
|
||||
|
||||
Reference in New Issue
Block a user