302 lines
12 KiB
Python
302 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from beaver.memory.runs import RunMemoryStore
|
|
from beaver.memory.skills import SkillDraftEvalReport, SkillLearningCandidate, SkillLearningStore
|
|
from beaver.skills.drafts import DraftService
|
|
from beaver.skills.learning import EvidenceSelector, SkillDraftSynthesizer, SkillLearningPipelineService, SkillLearningService
|
|
from beaver.skills.publisher import SkillPublisher
|
|
from beaver.skills.reviews import ReviewService
|
|
from beaver.skills.specs import SkillReviewState, SkillSpecStore
|
|
|
|
|
|
def _pipeline(tmp_path: Path) -> SkillLearningPipelineService:
|
|
spec_store = SkillSpecStore(tmp_path)
|
|
run_store = RunMemoryStore(tmp_path / "memory" / "runs")
|
|
learning_store = SkillLearningStore(tmp_path / "memory" / "skills")
|
|
draft_service = DraftService(spec_store)
|
|
learning_service = SkillLearningService(
|
|
run_store=run_store,
|
|
learning_store=learning_store,
|
|
draft_service=draft_service,
|
|
evidence_selector=EvidenceSelector(run_store),
|
|
synthesizer=SkillDraftSynthesizer(),
|
|
)
|
|
learning_store.record_learning_candidate(
|
|
SkillLearningCandidate(
|
|
candidate_id="candidate-1",
|
|
kind="retire_skill",
|
|
source_run_ids=["run-1"],
|
|
source_session_ids=["session-1"],
|
|
related_skill_names=["old-skill"],
|
|
reason="not useful",
|
|
evidence={"skill_version": "v0001"},
|
|
)
|
|
)
|
|
return SkillLearningPipelineService(
|
|
learning_store=learning_store,
|
|
learning_service=learning_service,
|
|
draft_service=draft_service,
|
|
review_service=ReviewService(spec_store),
|
|
publisher=SkillPublisher(spec_store),
|
|
)
|
|
|
|
|
|
def test_pipeline_lists_candidates_and_moves_draft_through_review(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="new-skill",
|
|
proposed_content="# New Skill\n\nDo the thing.",
|
|
proposed_frontmatter={"description": "test skill"},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
|
|
safety = pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
review = pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
version = pipeline.publish(draft.skill_name, draft.draft_id, publisher="tester")
|
|
|
|
assert pipeline.list_candidates()[0].candidate_id == "candidate-1"
|
|
assert review.status == SkillReviewState.IN_REVIEW.value
|
|
assert safety.passed is True
|
|
assert version.skill_name == "new-skill"
|
|
assert pipeline.get_draft(draft.skill_name, draft.draft_id).status == SkillReviewState.PUBLISHED.value
|
|
|
|
|
|
def test_pipeline_approve_requires_submitted_review(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="needs-review",
|
|
proposed_content="# Needs Review\n\nDo the thing.",
|
|
proposed_frontmatter={"description": "needs review"},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="in review before approval"):
|
|
pipeline.approve(draft.skill_name, draft.draft_id, reviewer="tester")
|
|
|
|
|
|
def test_pipeline_does_not_resubmit_terminal_draft(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="already-published",
|
|
proposed_content="# Already Published\n\nDo the thing.",
|
|
proposed_frontmatter={"description": "already published"},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
|
|
pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
pipeline.publish(draft.skill_name, draft.draft_id, publisher="tester")
|
|
|
|
with pytest.raises(ValueError, match="draft status before review submission"):
|
|
pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
|
|
|
|
def test_safety_recheck_keeps_submitted_candidate_in_review(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="reviewed-skill",
|
|
proposed_content="# Reviewed Skill\n\nDo the thing.",
|
|
proposed_frontmatter={"description": "reviewed"},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
candidate = pipeline.get_candidate("candidate-1")
|
|
candidate.draft_skill_name = draft.skill_name
|
|
candidate.draft_id = draft.draft_id
|
|
pipeline.learning_store.record_learning_candidate(candidate)
|
|
|
|
pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
|
|
assert pipeline.get_candidate("candidate-1").status == "review_pending"
|
|
|
|
|
|
def test_pipeline_reject_blocks_publish(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="blocked-skill",
|
|
proposed_content="# Blocked\n\nNo publish.",
|
|
proposed_frontmatter={"description": "blocked"},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
|
|
pipeline.reject(draft.skill_name, draft.draft_id, reviewer="tester")
|
|
|
|
with pytest.raises(ValueError, match="Draft not found"):
|
|
pipeline.publish(draft.skill_name, draft.draft_id, publisher="tester")
|
|
assert pipeline.draft_service.get_draft(draft.skill_name, draft.draft_id) is None
|
|
|
|
|
|
def test_pipeline_reject_removes_draft_from_review_list(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="remove-skill",
|
|
proposed_content="# Remove\n\nNo longer needed.",
|
|
proposed_frontmatter={"description": "remove"},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
|
|
review = pipeline.reject(draft.skill_name, draft.draft_id, reviewer="tester")
|
|
|
|
assert review.status == SkillReviewState.REJECTED.value
|
|
assert pipeline.list_drafts() == []
|
|
|
|
|
|
def test_publish_blocks_low_confidence_replay_report(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="low-confidence",
|
|
proposed_content="# Low\n\nDo it.",
|
|
proposed_frontmatter={"description": "low", "tools": []},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
pipeline.learning_store.write_eval_report(
|
|
SkillDraftEvalReport(
|
|
report_id="eval-low",
|
|
skill_name=draft.skill_name,
|
|
draft_id=draft.draft_id,
|
|
candidate_id="candidate-1",
|
|
passed=True,
|
|
baseline_score_avg=0.7,
|
|
candidate_score_avg=0.9,
|
|
score_delta=0.2,
|
|
regression_count=0,
|
|
improved_count=1,
|
|
unchanged_count=0,
|
|
confidence="low",
|
|
mode="replay",
|
|
eval_version="replay-v1",
|
|
execution_coverage=0.0,
|
|
surrogate_coverage=1.0,
|
|
blocked_coverage=0.0,
|
|
)
|
|
)
|
|
pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
|
|
with pytest.raises(ValueError, match="low confidence"):
|
|
pipeline.publish(draft.skill_name, draft.draft_id, publisher="tester")
|
|
|
|
|
|
def test_publish_blocks_failed_preservation_report(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_new_skill_draft(
|
|
skill_name="dropped-section",
|
|
proposed_content="# Skill\n\n## Workflow\n\nDo it.",
|
|
proposed_frontmatter={"description": "dropped", "tools": []},
|
|
created_by="test",
|
|
reason="test",
|
|
)
|
|
pipeline.learning_store.write_eval_report(
|
|
SkillDraftEvalReport(
|
|
report_id="eval-preservation",
|
|
skill_name=draft.skill_name,
|
|
draft_id=draft.draft_id,
|
|
candidate_id="candidate-1",
|
|
passed=True,
|
|
baseline_score_avg=0.7,
|
|
candidate_score_avg=0.9,
|
|
score_delta=0.2,
|
|
regression_count=0,
|
|
improved_count=1,
|
|
unchanged_count=0,
|
|
confidence="medium",
|
|
mode="replay",
|
|
eval_version="replay-v1",
|
|
preservation_report={"passed": False, "risk_level": "high", "dropped_sections": ["Safety"]},
|
|
)
|
|
)
|
|
pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
|
|
with pytest.raises(ValueError, match="preservation"):
|
|
pipeline.publish(draft.skill_name, draft.draft_id, publisher="tester")
|
|
|
|
|
|
def test_publish_blocks_plugin_three_way_without_plugin_preservation_report(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_plugin_update_draft(
|
|
skill_name="plugin-skill",
|
|
base_version="v0001",
|
|
proposed_content="# Plugin\n\nDo it.",
|
|
proposed_frontmatter={"description": "plugin", "tools": []},
|
|
created_by="test",
|
|
reason="plugin update",
|
|
provenance={"merge_mode": "three_way"},
|
|
)
|
|
pipeline.learning_store.write_eval_report(
|
|
SkillDraftEvalReport(
|
|
report_id="eval-plugin",
|
|
skill_name=draft.skill_name,
|
|
draft_id=draft.draft_id,
|
|
candidate_id="candidate-1",
|
|
passed=True,
|
|
baseline_score_avg=0.8,
|
|
candidate_score_avg=0.9,
|
|
score_delta=0.1,
|
|
regression_count=0,
|
|
improved_count=1,
|
|
unchanged_count=0,
|
|
confidence="medium",
|
|
mode="replay",
|
|
eval_version="replay-v1",
|
|
preservation_report={"passed": True, "mode": "ordinary"},
|
|
)
|
|
)
|
|
pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
|
|
with pytest.raises(ValueError, match="three-way preservation"):
|
|
pipeline.publish(draft.skill_name, draft.draft_id, publisher="tester")
|
|
|
|
|
|
def test_publish_blocks_plugin_update_with_unresolved_supporting_file_conflicts(tmp_path: Path) -> None:
|
|
pipeline = _pipeline(tmp_path)
|
|
draft = pipeline.draft_service.create_plugin_update_draft(
|
|
skill_name="plugin-skill",
|
|
base_version="v0001",
|
|
proposed_content="# Plugin\n\nDo it.",
|
|
proposed_frontmatter={"description": "plugin", "tools": []},
|
|
created_by="test",
|
|
reason="plugin update",
|
|
provenance={
|
|
"merge_mode": "three_way",
|
|
"supporting_file_plan": {"conflicts": [{"path": "a.txt", "reason": "diverged"}]},
|
|
},
|
|
)
|
|
pipeline.learning_store.write_eval_report(
|
|
SkillDraftEvalReport(
|
|
report_id="eval-plugin-conflict",
|
|
skill_name=draft.skill_name,
|
|
draft_id=draft.draft_id,
|
|
candidate_id="candidate-1",
|
|
passed=True,
|
|
baseline_score_avg=0.8,
|
|
candidate_score_avg=0.9,
|
|
score_delta=0.1,
|
|
regression_count=0,
|
|
improved_count=1,
|
|
unchanged_count=0,
|
|
confidence="medium",
|
|
mode="replay",
|
|
eval_version="replay-v1",
|
|
preservation_report={"passed": True, "mode": "plugin_three_way", "unresolved_conflicts": []},
|
|
)
|
|
)
|
|
pipeline.submit_review(draft.skill_name, draft.draft_id, requested_by="tester")
|
|
pipeline.check_safety(draft.skill_name, draft.draft_id)
|
|
|
|
with pytest.raises(ValueError, match="supporting-file conflicts"):
|
|
pipeline.publish(draft.skill_name, draft.draft_id, publisher="tester")
|