tomtan/EverOS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: initialize EverOS 1.0.0

Browse Source 
md-first memory extraction framework for AI agents.

Markdown is the single source of truth; SQLite holds state and LanceDB
provides the rebuildable vector + BM25 + scalar index. The codebase follows
a single-direction DDD layering (entrypoints -> service -> memory -> infra,
with component / core / config cross-cutting) enforced by import-linter.

Engineering surface:
- Coding conventions in .claude/rules/ (path-scoped) and workflows in
  .claude/skills/ (/commit, /new-branch, /pr).
- GitHub Actions CI runs make lint + test + integration; pre-commit mirrors
  the gates locally (ruff, hygiene hooks, gitlint commit-msg).
- Commit messages follow Conventional Commits, enforced by gitlint.
- make lint also enforces datetime two-zone discipline and OpenAPI drift.
This commit is contained in:
Elliot Chen
2026-06-05 22:35:51 +08:00
commit 518b8eca85
636 changed files with 160553 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
tests/unit/test_memory/test_cascade/__init__.py Normal file
View File

331
tests/unit/test_memory/test_cascade/test_handler_agent_skill.py Normal file
View File

@ -0,0 +1,331 @@
"""Tests for :class:`AgentSkillHandler` — whole-file skill reconcile.
Skill is the only kind that doesn't go through ``BaseDailyLogHandler``:
no entries, no per-entry diff. The digest is ``content_sha256`` over
the whole skill (name + description + body + references_content +
confidence + maturity_score); the handler reads ``SKILL.md`` + every
``references/*.md`` sibling and upserts one row per skill. These
tests build the directory layout on disk and verify the resulting
row + the delete path.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from everos.component.embedding import EmbeddingProvider
from everos.component.tokenizer import Tokenizer
from everos.core.persistence import MemoryRoot
from everos.infra.persistence.lancedb import AgentSkill
from everos.infra.persistence.markdown import AgentSkillWriter
from everos.memory.cascade.handlers import AgentSkillHandler, HandlerDeps
class _StubTokenizer(Tokenizer):
def tokenize(self, text: str) -> list[str]:
return [tok for tok in text.split() if tok]
def tokenize_batch(self, texts): # type: ignore[no-untyped-def]
return [self.tokenize(t) for t in texts]
class _StubEmbedder(EmbeddingProvider):
dim = 1024
async def embed(self, text: str) -> list[float]:
return [0.0] * self.dim
async def embed_batch(self, texts): # type: ignore[no-untyped-def]
return [await self.embed(t) for t in texts]
class _FakeSkillRepo:
def __init__(self) -> None:
self.rows: dict[str, AgentSkill] = {}
self.upserts: list[list[AgentSkill]] = []
self.deletes: list[str] = []
self.predicate_deletes: list[str] = []
async def get_by_id(self, row_id: str) -> AgentSkill | None:
return self.rows.get(row_id)
async def upsert(self, rows: list[AgentSkill]) -> None:
self.upserts.append(list(rows))
for row in rows:
self.rows[row.id] = row
async def delete_by_md_path(self, md_path: str) -> int:
self.deletes.append(md_path)
return 1
async def find_where(self, predicate: str, *, limit: int) -> list[AgentSkill]:
"""In-memory equivalent — handles only the
``md_path = '...' AND id != '...'`` shape the handler emits."""
if "md_path = " in predicate and "id != " in predicate:
md_lit = predicate.split("md_path = '")[1].split("'", 1)[0]
id_lit = predicate.split("id != '")[1].split("'", 1)[0]
return [
r for r in self.rows.values() if r.md_path == md_lit and r.id != id_lit
][:limit]
raise NotImplementedError(f"fake repo doesn't handle {predicate!r}")
async def delete(self, predicate: str) -> None:
self.predicate_deletes.append(predicate)
if "md_path = " in predicate and "id != " in predicate:
md_lit = predicate.split("md_path = '")[1].split("'", 1)[0]
id_lit = predicate.split("id != '")[1].split("'", 1)[0]
self.rows = {
rid: row
for rid, row in self.rows.items()
if not (row.md_path == md_lit and row.id != id_lit)
}
return
raise NotImplementedError(f"fake repo doesn't handle {predicate!r}")
@pytest.fixture
def memory_root(tmp_path: Path) -> MemoryRoot:
mr = MemoryRoot(tmp_path)
mr.ensure()
return mr
@pytest.fixture
def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeSkillRepo:
"""Patch the module-level repo the handler references."""
from everos.memory.cascade.handlers import agent_skill as skill_mod
repo = _FakeSkillRepo()
monkeypatch.setattr(skill_mod, "agent_skill_repo", repo)
return repo
async def _write_skill(
memory_root: MemoryRoot, agent_id: str, name: str, *, body: str
) -> str:
"""Create a SKILL.md via the real writer, return the relative md_path."""
from everos.infra.persistence.markdown import AgentSkillFrontmatter
writer = AgentSkillWriter(memory_root)
fm = AgentSkillFrontmatter(
id=f"skill_{name}",
agent_id=agent_id,
name=name,
description="Scan a contract draft for risk clauses.",
confidence=0.8,
maturity_score=0.6,
source_case_ids=["ac_1", "ac_2"],
)
await writer.write_main(agent_id, name, frontmatter=fm, body=body)
return f"default_app/default_project/agents/{agent_id}/skills/skill_{name}/SKILL.md"
async def test_handle_added_or_modified_upserts_typed_row(
memory_root: MemoryRoot, fake_repo: _FakeSkillRepo
) -> None:
md_path = await _write_skill(
memory_root, "a1", "contract_scan", body="step one\nstep two\n"
)
handler = AgentSkillHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
outcome = await handler.handle_added_or_modified(md_path)
assert outcome.upserted == 1
assert outcome.deleted == 0
row = fake_repo.upserts[0][0]
assert row.id == "a1_contract_scan"
assert row.owner_id == "a1"
assert row.owner_type == "agent"
assert row.name == "contract_scan"
assert row.description.startswith("Scan a contract draft")
assert row.description_tokens.startswith("Scan a contract draft")
assert row.confidence == pytest.approx(0.8)
assert row.maturity_score == pytest.approx(0.6)
assert row.source_case_ids == ["ac_1", "ac_2"]
assert row.md_path == md_path
assert len(row.vector) == 1024
# Body content lands in the ``content`` column.
assert "step one" in row.content
assert "step two" in row.content
async def test_references_md_concatenated_into_content(
memory_root: MemoryRoot, fake_repo: _FakeSkillRepo
) -> None:
"""references/*.md siblings are appended to ``content`` deterministically."""
md_path = await _write_skill(memory_root, "a1", "skill_x", body="main body text")
# Drop two reference files into the skill dir.
refs_dir = (
memory_root.root
/ "default_app/default_project/agents/a1/skills/skill_skill_x/references"
)
refs_dir.mkdir(parents=True, exist_ok=True)
(refs_dir / "b.md").write_text("reference B content\n", encoding="utf-8")
(refs_dir / "a.md").write_text("reference A content\n", encoding="utf-8")
handler = AgentSkillHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
await handler.handle_added_or_modified(md_path)
content = fake_repo.upserts[0][0].content
# Body comes first, references sorted by filename (a.md then b.md).
assert content.index("main body text") < content.index("reference A content")
assert content.index("reference A content") < content.index("reference B content")
async def test_renaming_skill_via_frontmatter_clears_old_row(
memory_root: MemoryRoot, fake_repo: _FakeSkillRepo
) -> None:
"""User edits SKILL.md frontmatter.name; the LanceDB row id changes.
skill_id is derived from ``frontmatter.name`` (``<owner_id>_<name>``).
When the user edits the name in place — common when refining a skill
title without moving the file — the new id differs from the old, so
a plain ``upsert([new_row])`` would leave the old row behind and a
subsequent search would return both. The handler must sweep the
stale row by ``md_path = ? AND id != new_id`` before the upsert.
"""
# First pass: write the original SKILL.md and let cascade index it.
md_path = await _write_skill(memory_root, "a1", "old_name", body="step one\n")
handler = AgentSkillHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
await handler.handle_added_or_modified(md_path)
assert fake_repo.rows == {"a1_old_name": fake_repo.rows["a1_old_name"]}
# Second pass: simulate the user editing frontmatter.name in place
# (md_path unchanged, only the name field flips).
absolute = memory_root.root / md_path
text = absolute.read_text(encoding="utf-8")
absolute.write_text(text.replace("name: old_name", "name: new_name"))
outcome = await handler.handle_added_or_modified(md_path)
assert outcome.upserted == 1
assert outcome.deleted == 1
# Old id is gone, new id is present, exactly one row survives.
assert list(fake_repo.rows.keys()) == ["a1_new_name"]
# The sweep predicate references the *new* id with the same md_path.
assert fake_repo.predicate_deletes == [
f"md_path = '{md_path}' AND id != 'a1_new_name'"
]
async def test_first_create_does_not_call_orphan_sweep(
memory_root: MemoryRoot, fake_repo: _FakeSkillRepo
) -> None:
"""First write of a SKILL.md issues an upsert but no orphan delete.
The sweep clause only kicks in when there's a prior row at the same
md_path under a different id (the rename case). For a fresh skill
we should not bother LanceDB with an empty delete predicate either.
"""
md_path = await _write_skill(memory_root, "a1", "fresh_skill", body="x")
handler = AgentSkillHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
outcome = await handler.handle_added_or_modified(md_path)
assert outcome.upserted == 1
assert outcome.deleted == 0
# The handler does call find_where on first-pass (prior is None),
# but the empty result short-circuits the delete.
assert fake_repo.predicate_deletes == []
async def test_content_edit_skips_orphan_lookup(
memory_root: MemoryRoot, fake_repo: _FakeSkillRepo
) -> None:
"""When the name is unchanged (prior row exists under the same id),
the handler must not pay for the orphan find — there can't be any.
"""
md_path = await _write_skill(memory_root, "a1", "stable_name", body="v1\n")
handler = AgentSkillHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
await handler.handle_added_or_modified(md_path)
# Edit the body so digest drifts (forces upsert path, not skip).
absolute = memory_root.root / md_path
absolute.write_text(
absolute.read_text(encoding="utf-8").replace("v1", "v2"),
encoding="utf-8",
)
outcome = await handler.handle_added_or_modified(md_path)
assert outcome.upserted == 1
assert outcome.deleted == 0
# Same id, no orphan sweep issued.
assert fake_repo.predicate_deletes == []
async def test_handle_deleted_calls_delete_by_md_path(
memory_root: MemoryRoot, fake_repo: _FakeSkillRepo
) -> None:
handler = AgentSkillHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
outcome = await handler.handle_deleted("agents/a1/skills/skill_x/SKILL.md")
assert outcome.deleted == 1
assert outcome.upserted == 0
assert fake_repo.deletes == ["agents/a1/skills/skill_x/SKILL.md"]
async def test_missing_name_raises(
memory_root: MemoryRoot, fake_repo: _FakeSkillRepo
) -> None:
"""A SKILL.md whose frontmatter lacks ``name`` surfaces as ValueError."""
# Hand-write a malformed SKILL.md (no `name`).
skill_dir = memory_root.root / "agents/a1/skills/skill_broken"
skill_dir.mkdir(parents=True, exist_ok=True)
(skill_dir / "SKILL.md").write_text(
"---\n"
"id: skill_broken\n"
"type: agent_skill\n"
"agent_id: a1\n"
"track: agent\n"
"description: x\n"
"confidence: 0.5\n"
"maturity_score: 0.5\n"
"---\nbody\n",
encoding="utf-8",
)
handler = AgentSkillHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
with pytest.raises(ValueError, match="name"):
await handler.handle_added_or_modified("agents/a1/skills/skill_broken/SKILL.md")

260
tests/unit/test_memory/test_cascade/test_handler_episode.py Normal file
View File

@ -0,0 +1,260 @@
"""Tests for :class:`EpisodeHandler` — md → LanceDB row reconcile.
Uses a real on-disk md file (via :class:`EpisodeWriter`) to exercise
the parse → diff → upsert path. The lancedb repo is faked since the
production singleton would need a live LanceDB connection; this keeps
the test in-memory while still validating row construction and the
3-way diff branch behaviour.
"""
from __future__ import annotations
import datetime as _dt
from pathlib import Path
from typing import Any
import pytest
from everos.component.embedding import EmbeddingProvider
from everos.component.tokenizer import Tokenizer
from everos.core.persistence import MemoryRoot
from everos.infra.persistence.lancedb import Episode
from everos.infra.persistence.markdown import EpisodeWriter
from everos.memory.cascade.handlers import HandlerDeps
from everos.memory.cascade.handlers.episode import EpisodeHandler
class _StubTokenizer(Tokenizer):
"""Returns the input split on whitespace — deterministic for assertions."""
def tokenize(self, text: str) -> list[str]:
return [tok for tok in text.split() if tok]
def tokenize_batch(self, texts): # type: ignore[no-untyped-def]
return [self.tokenize(t) for t in texts]
class _StubEmbedder(EmbeddingProvider):
"""Returns a fixed 1024-dim vector; records call count."""
dim = 1024
def __init__(self) -> None:
self.calls = 0
async def embed(self, text: str) -> list[float]:
self.calls += 1
return [0.1] * self.dim
async def embed_batch(self, texts): # type: ignore[no-untyped-def]
return [await self.embed(t) for t in texts]
class _FakeEpisodeRepo:
"""Recording repo — captures upserts / deletes the handler issues."""
def __init__(self) -> None:
self.upserts: list[list[Episode]] = []
self.deletes: list[str] = []
self.rows: list[Episode] = []
async def find_where(self, where: str, *, limit: int = 100) -> list[Episode]:
# Honour only the md_path = '...' filter the handler emits.
prefix = "md_path = '"
if where.startswith(prefix):
md_path = where[len(prefix) :].rstrip("'")
return [r for r in self.rows if r.md_path == md_path]
return []
async def upsert(self, rows: list[Episode]) -> None:
self.upserts.append(list(rows))
# Reflect into ``self.rows`` so a follow-up find_where sees the state.
by_id = {r.id: r for r in self.rows}
for r in rows:
by_id[r.id] = r
self.rows = list(by_id.values())
async def delete(self, predicate: str) -> None:
self.deletes.append(predicate)
async def delete_by_md_path(self, md_path: str) -> int:
before = len(self.rows)
self.rows = [r for r in self.rows if r.md_path != md_path]
return before - len(self.rows)
@pytest.fixture
def memory_root(tmp_path: Path) -> MemoryRoot:
mr = MemoryRoot(tmp_path)
mr.ensure()
return mr
@pytest.fixture
def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeEpisodeRepo:
"""Swap the class-level ``lance_repo`` on EpisodeHandler.
After the BaseDailyLogHandler refactor, the repo binding is a
ClassVar resolved at class-definition time; patching the module
attribute would no longer reach the handler's call sites.
"""
from everos.memory.cascade.handlers.episode import EpisodeHandler
repo = _FakeEpisodeRepo()
monkeypatch.setattr(EpisodeHandler, "lance_repo", repo)
return repo
async def _write_one_entry(writer: EpisodeWriter, owner_id: str, body: str) -> str:
"""Append a single episode entry, return the md path (relative)."""
today = _dt.date(2026, 5, 14)
await writer.append_entry(
owner_id,
inline={
"owner_id": owner_id,
"session_id": "s1",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_type": "memcell",
"parent_id": "mc_test_parent",
"sender_ids": [owner_id],
},
sections={"Subject": "Test", "Summary": "Stub", "Content": body},
date=today,
)
return (
f"default_app/default_project/users/{owner_id}/episodes/episode-2026-05-14.md"
)
def _build_handler(
memory_root: MemoryRoot,
) -> tuple[EpisodeHandler, _StubEmbedder]:
embedder = _StubEmbedder()
deps = HandlerDeps(
memory_root=memory_root,
embedder=embedder,
tokenizer=_StubTokenizer(),
)
return EpisodeHandler(deps), embedder
# ── happy path ───────────────────────────────────────────────────────────
async def test_added_entry_upserts_typed_row(
memory_root: MemoryRoot, fake_repo: _FakeEpisodeRepo
) -> None:
writer = EpisodeWriter(memory_root)
rel = await _write_one_entry(writer, "u1", "hello world")
handler, embedder = _build_handler(memory_root)
outcome = await handler.handle_added_or_modified(rel)
assert outcome.upserted == 1
assert outcome.deleted == 0
assert outcome.skipped == 0
assert embedder.calls == 1
assert len(fake_repo.upserts) == 1
row = fake_repo.upserts[0][0]
assert row.owner_id == "u1"
assert row.owner_type == "user"
# Scope is parsed back from the md path's <app>/<project> prefix.
assert row.app_id == "default"
assert row.project_id == "default"
assert row.session_id == "s1"
assert row.parent_id == "mc_test_parent"
assert row.parent_type == "memcell"
assert row.episode == "hello world"
assert row.episode_tokens == "hello world"
assert row.subject == "Test"
assert row.md_path == rel
assert row.entry_id.startswith("ep_")
assert row.id == f"u1_{row.entry_id}"
assert len(row.vector) == 1024
async def test_unchanged_entry_is_skipped_no_embed_call(
memory_root: MemoryRoot, fake_repo: _FakeEpisodeRepo
) -> None:
"""Second handle run with no md change → skipped + no embed call."""
writer = EpisodeWriter(memory_root)
rel = await _write_one_entry(writer, "u1", "hello world")
handler, embedder = _build_handler(memory_root)
await handler.handle_added_or_modified(rel) # first pass populates fake repo
fake_repo.upserts.clear()
embedder.calls = 0
outcome = await handler.handle_added_or_modified(rel)
assert outcome.skipped == 1
assert outcome.upserted == 0
assert embedder.calls == 0
assert fake_repo.upserts == []
async def test_modified_entry_reembeds(
memory_root: MemoryRoot, fake_repo: _FakeEpisodeRepo
) -> None:
"""Changing the entry body bumps the sha → re-embed + upsert."""
writer = EpisodeWriter(memory_root)
rel = await _write_one_entry(writer, "u1", "original content")
handler, embedder = _build_handler(memory_root)
await handler.handle_added_or_modified(rel)
# Tamper with the row's stored sha so the next pass sees a mismatch.
fake_repo.rows[0] = fake_repo.rows[0].model_copy(
update={"content_sha256": "0" * 64}
)
fake_repo.upserts.clear()
embedder.calls = 0
outcome = await handler.handle_added_or_modified(rel)
assert outcome.upserted == 1
assert outcome.skipped == 0
assert embedder.calls == 1
# ── deletion paths ───────────────────────────────────────────────────────
async def test_handle_deleted_wipes_md_path_rows(
memory_root: MemoryRoot, fake_repo: _FakeEpisodeRepo
) -> None:
writer = EpisodeWriter(memory_root)
rel = await _write_one_entry(writer, "u1", "hello")
handler, _embedder = _build_handler(memory_root)
await handler.handle_added_or_modified(rel)
assert fake_repo.rows # populated
outcome = await handler.handle_deleted(rel)
assert outcome.deleted == 1
assert fake_repo.rows == []
# ── error path ───────────────────────────────────────────────────────────
async def test_missing_timestamp_raises_value_error(
memory_root: MemoryRoot, fake_repo: _FakeEpisodeRepo
) -> None:
"""Malformed inline surfaces as ValueError — worker treats unrecoverable."""
writer = EpisodeWriter(memory_root)
# Manually bypass the writer to drop timestamp.
today = _dt.date(2026, 5, 14)
await writer.append_entry(
"u1",
inline={"owner_id": "u1", "session_id": "s1"}, # no timestamp
sections={"Content": "x"},
date=today,
)
rel = "default_app/default_project/users/u1/episodes/episode-2026-05-14.md"
handler, _embedder = _build_handler(memory_root)
with pytest.raises(ValueError, match="timestamp"):
await handler.handle_added_or_modified(rel)
# ── unused noqa suppressor (keep imports tidy) ──────────────────────────
_: Any = None

260
tests/unit/test_memory/test_cascade/test_handler_user_profile.py Normal file
View File

@ -0,0 +1,260 @@
"""Tests for :class:`UserProfileHandler` — single-file profile reconcile.
UserProfile is the second single-file kind (after AgentSkill) — one
``users/<user_id>/user.md`` per user, replaced wholesale on edit. The
handler upserts one row per profile and skips when the
content-bearing digest (summary + JSON buckets) is unchanged. These
tests verify the upsert / skip path, the JSON encoding of
``explicit_info`` / ``implicit_traits``, and the missing-user_id guard.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from everos.component.embedding import EmbeddingProvider
from everos.component.tokenizer import Tokenizer
from everos.core.persistence import MemoryRoot
from everos.infra.persistence.lancedb import UserProfile
from everos.infra.persistence.markdown import ProfileWriter, UserProfileFrontmatter
from everos.memory.cascade.handlers import HandlerDeps, UserProfileHandler
class _StubTokenizer(Tokenizer):
def tokenize(self, text: str) -> list[str]:
return [tok for tok in text.split() if tok]
def tokenize_batch(self, texts): # type: ignore[no-untyped-def]
return [self.tokenize(t) for t in texts]
class _StubEmbedder(EmbeddingProvider):
"""Profile handler does not embed; the stub stays as a no-op so the
shared :class:`HandlerDeps` shape is satisfied."""
dim = 1024
async def embed(self, text: str) -> list[float]: # pragma: no cover
raise AssertionError("UserProfileHandler must not call the embedder")
async def embed_batch( # pragma: no cover
self,
texts, # type: ignore[no-untyped-def]
):
raise AssertionError("UserProfileHandler must not call the embedder")
class _FakeProfileRepo:
def __init__(self) -> None:
self.rows: dict[str, UserProfile] = {}
self.upserts: list[list[UserProfile]] = []
self.deletes: list[str] = []
async def get_by_id(self, row_id: str) -> UserProfile | None:
return self.rows.get(row_id)
async def upsert(self, rows: list[UserProfile]) -> None:
self.upserts.append(list(rows))
for row in rows:
self.rows[row.id] = row
async def delete_by_md_path(self, md_path: str) -> int:
self.deletes.append(md_path)
before = len(self.rows)
self.rows = {rid: r for rid, r in self.rows.items() if r.md_path != md_path}
return before - len(self.rows)
@pytest.fixture
def memory_root(tmp_path: Path) -> MemoryRoot:
mr = MemoryRoot(tmp_path)
mr.ensure()
return mr
@pytest.fixture
def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeProfileRepo:
from everos.memory.cascade.handlers import user_profile as up_mod
repo = _FakeProfileRepo()
monkeypatch.setattr(up_mod, "user_profile_repo", repo)
return repo
async def _write_profile(
memory_root: MemoryRoot,
user_id: str,
*,
summary: str,
explicit_info: list,
implicit_traits: list,
profile_timestamp_ms: int = 1_700_000_000_000,
) -> str:
writer = ProfileWriter(memory_root)
fm = UserProfileFrontmatter(
id=f"user_profile_{user_id}",
user_id=user_id,
summary=summary,
explicit_info=explicit_info,
implicit_traits=implicit_traits,
profile_timestamp_ms=profile_timestamp_ms,
)
await writer.write(user_id, frontmatter=fm, body="display text")
return f"default_app/default_project/users/{user_id}/user.md"
def _handler(memory_root: MemoryRoot) -> UserProfileHandler:
return UserProfileHandler(
HandlerDeps(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
)
async def test_first_pass_upserts_typed_row(
memory_root: MemoryRoot, fake_repo: _FakeProfileRepo
) -> None:
md_path = await _write_profile(
memory_root,
"u_alice",
summary="Alice likes long hikes and prefers oat milk.",
explicit_info=[{"fact": "lives in tokyo"}, "renew passport"],
implicit_traits=[{"trait": "introverted"}],
)
outcome = await _handler(memory_root).handle_added_or_modified(md_path)
assert outcome.upserted == 1
assert outcome.skipped == 0
row = fake_repo.upserts[0][0]
assert row.id == "u_alice"
assert row.owner_id == "u_alice"
assert row.owner_type == "user"
assert row.summary.startswith("Alice")
assert row.md_path == md_path
# Heterogeneous buckets land as canonical JSON strings.
assert json.loads(row.explicit_info_json) == [
{"fact": "lives in tokyo"},
"renew passport",
]
assert json.loads(row.implicit_traits_json) == [{"trait": "introverted"}]
assert row.profile_timestamp_ms == 1_700_000_000_000
async def test_second_pass_with_same_content_skips(
memory_root: MemoryRoot, fake_repo: _FakeProfileRepo
) -> None:
md_path = await _write_profile(
memory_root,
"u_alice",
summary="Stable summary.",
explicit_info=["a"],
implicit_traits=["b"],
)
handler = _handler(memory_root)
first = await handler.handle_added_or_modified(md_path)
assert first.upserted == 1
# Re-run with no edits — digest matches, handler must skip.
second = await handler.handle_added_or_modified(md_path)
assert second.upserted == 0
assert second.skipped == 1
# Only the first pass touched the repo.
assert len(fake_repo.upserts) == 1
async def test_timestamp_only_drift_skips(
memory_root: MemoryRoot, fake_repo: _FakeProfileRepo
) -> None:
"""Re-synthesis bumps ``profile_timestamp_ms`` even when the content
is byte-identical; the digest excludes the timestamp so cascade
skips re-upsert and avoids a wasted index write."""
md_path = await _write_profile(
memory_root,
"u_alice",
summary="Same summary.",
explicit_info=["x"],
implicit_traits=["y"],
profile_timestamp_ms=1_700_000_000_000,
)
handler = _handler(memory_root)
await handler.handle_added_or_modified(md_path)
# Bump only profile_timestamp_ms.
absolute = memory_root.root / md_path
absolute.write_text(
absolute.read_text(encoding="utf-8").replace("1700000000000", "1800000000000"),
encoding="utf-8",
)
outcome = await handler.handle_added_or_modified(md_path)
assert outcome.upserted == 0
assert outcome.skipped == 1
async def test_summary_edit_triggers_upsert(
memory_root: MemoryRoot, fake_repo: _FakeProfileRepo
) -> None:
md_path = await _write_profile(
memory_root,
"u_alice",
summary="Original summary.",
explicit_info=[],
implicit_traits=[],
)
handler = _handler(memory_root)
await handler.handle_added_or_modified(md_path)
assert len(fake_repo.upserts) == 1
absolute = memory_root.root / md_path
absolute.write_text(
absolute.read_text(encoding="utf-8").replace(
"Original summary.", "New shiny summary."
),
encoding="utf-8",
)
outcome = await handler.handle_added_or_modified(md_path)
assert outcome.upserted == 1
assert fake_repo.upserts[1][0].summary == "New shiny summary."
async def test_missing_user_id_raises(
memory_root: MemoryRoot, fake_repo: _FakeProfileRepo
) -> None:
bad_dir = memory_root.root / "users" / "u_x"
bad_dir.mkdir(parents=True, exist_ok=True)
(bad_dir / "user.md").write_text(
"---\n"
"id: user_profile_u_x\n"
"type: user_profile\n"
"track: user\n"
"summary: x\n"
"---\nbody\n",
encoding="utf-8",
)
with pytest.raises(ValueError, match="user_id"):
await _handler(memory_root).handle_added_or_modified("users/u_x/user.md")
async def test_handle_deleted_drops_row(
memory_root: MemoryRoot, fake_repo: _FakeProfileRepo
) -> None:
md_path = await _write_profile(
memory_root,
"u_alice",
summary="bye",
explicit_info=[],
implicit_traits=[],
)
handler = _handler(memory_root)
await handler.handle_added_or_modified(md_path)
assert "u_alice" in fake_repo.rows
outcome = await handler.handle_deleted(md_path)
assert outcome.deleted == 1
assert fake_repo.deletes == [md_path]
assert "u_alice" not in fake_repo.rows

261
tests/unit/test_memory/test_cascade/test_handlers_daily_log_mapping.py Normal file
View File

@ -0,0 +1,261 @@
"""Per-kind ``_build_row`` mapping for the 3 non-Episode daily-log handlers.
The diff loop (read → sha256 → 3-way diff → upsert/delete) lives on
:class:`BaseDailyLogHandler` and is exercised by
``test_handler_episode.py``. These tests focus on the kind-specific
:meth:`_build_row` mapping — given a synthesised ``ParsedEntry``, do
the right LanceDB columns get populated?
Each kind gets one happy-path test (all fields present) plus a
focused error-path test (missing required inline field). Sharing one
file avoids 3 nearly-identical fixture stacks.
"""
from __future__ import annotations
import datetime as _dt
import pytest
from everos.component.embedding import EmbeddingProvider
from everos.component.tokenizer import Tokenizer
from everos.core.persistence import MemoryRoot, StructuredEntry
from everos.memory.cascade.handlers import (
AgentCaseHandler,
AtomicFactHandler,
ForesightHandler,
HandlerDeps,
)
from everos.memory.cascade.handlers._daily_log_base import ParsedEntry
class _StubTokenizer(Tokenizer):
def tokenize(self, text: str) -> list[str]:
return [tok for tok in text.split() if tok]
def tokenize_batch(self, texts): # type: ignore[no-untyped-def]
return [self.tokenize(t) for t in texts]
class _StubEmbedder(EmbeddingProvider):
dim = 1024
async def embed(self, text: str) -> list[float]:
return [0.0] * self.dim
async def embed_batch(self, texts): # type: ignore[no-untyped-def]
return [await self.embed(t) for t in texts]
def _deps(tmp_path) -> HandlerDeps: # type: ignore[no-untyped-def]
mr = MemoryRoot(tmp_path)
mr.ensure()
return HandlerDeps(
memory_root=mr,
embedder=_StubEmbedder(),
tokenizer=_StubTokenizer(),
)
def _entry(
entry_id: str,
inline: dict[str, str],
sections: dict[str, str],
*,
sha: str = "f" * 64,
) -> ParsedEntry:
return ParsedEntry(
entry_id=entry_id,
structured=StructuredEntry(
id=entry_id,
body="",
start=0,
end=0,
header=None,
inline=inline,
sections=sections,
),
content_sha256=sha,
)
# ── AtomicFact ───────────────────────────────────────────────────────────
async def test_atomic_fact_build_row_maps_inline_and_section(tmp_path) -> None: # type: ignore[no-untyped-def]
handler = AtomicFactHandler(_deps(tmp_path))
row = await handler._build_row(
owner_id="u1",
owner_type="user",
md_path="users/u1/.atomic_facts/atomic_fact-2026-05-14.md",
entry=_entry(
"af_20260514_0001",
inline={
"owner_id": "u1",
"session_id": "s1",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_id": "mc_1",
"sender_ids": "[u1, u2]",
},
sections={"Fact": "the user prefers dark mode"},
),
)
assert row.id == "u1_af_20260514_0001"
assert row.fact == "the user prefers dark mode"
assert row.fact_tokens == "the user prefers dark mode"
assert row.parent_id == "mc_1"
assert row.sender_ids == ["u1", "u2"]
assert row.timestamp == _dt.datetime(2026, 5, 14, 10, 0, tzinfo=_dt.UTC)
assert row.md_path.endswith("atomic_fact-2026-05-14.md")
assert len(row.vector) == 1024
async def test_atomic_fact_missing_timestamp_raises(tmp_path) -> None: # type: ignore[no-untyped-def]
handler = AtomicFactHandler(_deps(tmp_path))
with pytest.raises(ValueError, match="timestamp"):
await handler._build_row(
owner_id="u1",
owner_type="user",
md_path="x.md",
entry=_entry(
"af_20260514_0001",
inline={"owner_id": "u1", "session_id": "s1"},
sections={"Fact": "x"},
),
)
# ── Foresight ────────────────────────────────────────────────────────────
async def test_foresight_build_row_with_evidence(tmp_path) -> None: # type: ignore[no-untyped-def]
handler = ForesightHandler(_deps(tmp_path))
row = await handler._build_row(
owner_id="u1",
owner_type="user",
md_path="users/u1/.foresights/foresight-2026-05-14.md",
entry=_entry(
"fs_20260514_0001",
inline={
"owner_id": "u1",
"session_id": "s1",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_id": "mc_1",
"start_time": "2026-05-14T11:00:00+00:00",
"end_time": "2026-05-14T13:00:00+00:00",
"duration_days": "2",
},
sections={
"Foresight": "user will book lunch",
"Evidence": "calendar invite mentions 12pm",
},
),
)
assert row.foresight == "user will book lunch"
assert row.foresight_tokens == "user will book lunch"
assert row.evidence == "calendar invite mentions 12pm"
assert row.evidence_tokens == "calendar invite mentions 12pm"
assert row.start_time == _dt.datetime(2026, 5, 14, 11, 0, tzinfo=_dt.UTC)
assert row.end_time == _dt.datetime(2026, 5, 14, 13, 0, tzinfo=_dt.UTC)
assert row.duration_days == 2
async def test_foresight_optional_evidence_left_none(tmp_path) -> None: # type: ignore[no-untyped-def]
handler = ForesightHandler(_deps(tmp_path))
row = await handler._build_row(
owner_id="u1",
owner_type="user",
md_path="x.md",
entry=_entry(
"fs_20260514_0001",
inline={
"owner_id": "u1",
"session_id": "s1",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_id": "mc_1",
},
sections={"Foresight": "user will book lunch"},
),
)
assert row.evidence is None
assert row.evidence_tokens is None
assert row.start_time is None
assert row.end_time is None
assert row.duration_days is None
# ── AgentCase ────────────────────────────────────────────────────────────
async def test_agent_case_build_row_maps_intent_approach_insight(tmp_path) -> None: # type: ignore[no-untyped-def]
handler = AgentCaseHandler(_deps(tmp_path))
row = await handler._build_row(
owner_id="a1",
owner_type="agent",
md_path="agents/a1/.cases/agent_case-2026-05-14.md",
entry=_entry(
"ac_20260514_0001",
inline={
"owner_id": "a1",
"session_id": "s1",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_id": "mc_1",
"quality_score": "0.87",
},
sections={
"TaskIntent": "scan contract for risk clauses",
"Approach": "1. read pages 1-5; 2. flag indemnity",
"KeyInsight": "indemnity cap missing",
},
),
)
assert row.task_intent == "scan contract for risk clauses"
assert row.task_intent_tokens == "scan contract for risk clauses"
assert row.approach.startswith("1. read pages")
assert row.key_insight == "indemnity cap missing"
assert row.quality_score == pytest.approx(0.87)
assert row.owner_type == "agent"
async def test_agent_case_optional_insight_left_none(tmp_path) -> None: # type: ignore[no-untyped-def]
handler = AgentCaseHandler(_deps(tmp_path))
row = await handler._build_row(
owner_id="a1",
owner_type="agent",
md_path="x.md",
entry=_entry(
"ac_20260514_0001",
inline={
"owner_id": "a1",
"session_id": "s1",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_id": "mc_1",
"quality_score": "0.5",
},
sections={
"TaskIntent": "x",
"Approach": "y",
},
),
)
assert row.key_insight is None
async def test_agent_case_missing_quality_score_raises(tmp_path) -> None: # type: ignore[no-untyped-def]
handler = AgentCaseHandler(_deps(tmp_path))
with pytest.raises(ValueError, match="quality_score"):
await handler._build_row(
owner_id="a1",
owner_type="agent",
md_path="x.md",
entry=_entry(
"ac_20260514_0001",
inline={
"owner_id": "a1",
"session_id": "s1",
"timestamp": "2026-05-14T10:00:00+00:00",
"parent_id": "mc_1",
},
sections={"TaskIntent": "x", "Approach": "y"},
),
)

106
tests/unit/test_memory/test_cascade/test_orchestrator.py Normal file
View File

@ -0,0 +1,106 @@
"""``CascadeOrchestrator`` — idempotent start/stop, queue_summary forwards."""
from __future__ import annotations
from collections.abc import AsyncIterator
from pathlib import Path
import pytest
from sqlmodel import SQLModel
from everos.component.embedding import EmbeddingProvider
from everos.component.tokenizer import build_tokenizer
from everos.core.persistence import MemoryRoot
from everos.infra.persistence.lancedb import (
dispose_connection,
ensure_business_indexes,
)
from everos.infra.persistence.sqlite import dispose_engine, get_engine
from everos.memory.cascade import CascadeConfig, CascadeOrchestrator
class _StubEmbedder(EmbeddingProvider):
dim = 1024
async def embed(self, text: str) -> list[float]:
return [0.0] * self.dim
async def embed_batch(self, texts): # type: ignore[no-untyped-def]
return [[0.0] * self.dim for _ in texts]
@pytest.fixture
async def runtime(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> AsyncIterator[MemoryRoot]:
"""Boot sqlite + lancedb against a tmp memory_root."""
monkeypatch.setenv("EVEROS_MEMORY__ROOT", str(tmp_path))
monkeypatch.setenv("EVEROS_EMBEDDING__MODEL", "stub-model")
monkeypatch.setenv("EVEROS_EMBEDDING__BASE_URL", "http://stub.invalid/v1")
monkeypatch.setenv("EVEROS_EMBEDDING__API_KEY", "stub-key")
await dispose_connection()
await dispose_engine()
engine = get_engine()
async with engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await ensure_business_indexes()
yield MemoryRoot.default()
await dispose_connection()
await dispose_engine()
def _make_orchestrator(memory_root: MemoryRoot) -> CascadeOrchestrator:
return CascadeOrchestrator(
memory_root=memory_root,
embedder=_StubEmbedder(),
tokenizer=build_tokenizer(),
config=CascadeConfig(
scan_interval_seconds=60.0,
worker_batch_size=10,
worker_max_retry=1,
worker_poll_interval_seconds=0.05,
worker_retry_backoff_seconds=0.0,
),
)
async def test_double_start_is_idempotent(runtime: MemoryRoot) -> None:
"""Calling start twice does not relaunch tasks."""
orch = _make_orchestrator(runtime)
await orch.start()
# Capture watcher identity to verify the second start doesn't replace it.
first_watcher = orch._watcher
await orch.start()
assert orch._watcher is first_watcher
await orch.stop()
async def test_stop_before_start_is_noop(runtime: MemoryRoot) -> None:
orch = _make_orchestrator(runtime)
await orch.stop() # must not raise; nothing to do
async def test_double_stop_is_idempotent(runtime: MemoryRoot) -> None:
orch = _make_orchestrator(runtime)
await orch.start()
await orch.stop()
await orch.stop() # second stop is a no-op
async def test_queue_summary_returns_empty_on_fresh_runtime(
runtime: MemoryRoot,
) -> None:
orch = _make_orchestrator(runtime)
summary = await orch.queue_summary()
assert summary.pending == 0
assert summary.done == 0
assert summary.failed_retryable == 0
assert summary.failed_permanent == 0
async def test_drain_once_returns_zero_on_empty_queue(
runtime: MemoryRoot,
) -> None:
orch = _make_orchestrator(runtime)
assert await orch.drain_once() == 0

137
tests/unit/test_memory/test_cascade/test_reconciler.py Normal file
View File

@ -0,0 +1,137 @@
"""Tests for :func:`reconcile` — pure scan vs state diff.
The reconciler is pure (no IO), so each scenario is just a few
dataclass instances in / decisions out. Covers the 4 cases:
``added`` / ``modified`` / ``deleted`` / ``no-op``.
"""
from __future__ import annotations
from everos.memory.cascade.reconciler import PriorState, reconcile
from everos.memory.cascade.types import ScanInput
def _scan(path: str, mtime: float = 1.0, kind: str = "episode") -> ScanInput:
return ScanInput(md_path=path, mtime=mtime, kind=kind)
def _state(
path: str,
*,
mtime: float = 1.0,
kind: str = "episode",
status: str = "done",
change_type: str = "modified",
) -> PriorState:
return PriorState(
md_path=path,
kind=kind,
mtime=mtime,
status=status,
change_type=change_type,
)
def test_added_path_emits_added_decision() -> None:
decisions = reconcile([_scan("a.md")], state={})
assert [(d.md_path, d.change_type) for d in decisions] == [("a.md", "added")]
def test_modified_mtime_emits_modified_decision() -> None:
decisions = reconcile(
[_scan("a.md", mtime=2.0)],
state={"a.md": _state("a.md", mtime=1.0)},
)
assert [(d.md_path, d.change_type) for d in decisions] == [("a.md", "modified")]
def test_done_state_with_matching_mtime_is_skipped() -> None:
"""Quiet sweeps must stay quiet — no upsert churn."""
decisions = reconcile(
[_scan("a.md", mtime=1.0)],
state={"a.md": _state("a.md", mtime=1.0, status="done")},
)
assert decisions == []
def test_pending_state_with_matching_mtime_still_emits_modified() -> None:
"""Pending / failed states are NOT terminal — re-emit so worker re-runs."""
decisions = reconcile(
[_scan("a.md", mtime=1.0)],
state={"a.md": _state("a.md", mtime=1.0, status="pending")},
)
assert [(d.md_path, d.change_type) for d in decisions] == [("a.md", "modified")]
def test_deleted_path_emits_deleted_decision() -> None:
decisions = reconcile(
[],
state={"a.md": _state("a.md", status="pending")},
)
assert [(d.md_path, d.change_type) for d in decisions] == [("a.md", "deleted")]
def test_deleted_path_already_done_as_delete_is_skipped() -> None:
"""A done row that is itself a successful delete cycle — don't re-emit."""
decisions = reconcile(
[],
state={
"a.md": _state("a.md", status="done", change_type="deleted"),
},
)
assert decisions == []
def test_done_added_row_with_missing_path_is_recovered_as_deleted() -> None:
"""Watcher missed an unlink (e.g. fseventsd drop / daemon restart).
The state row is ``status='done'`` from the previous add cycle, but
the file is gone from disk. The scanner MUST re-emit a 'deleted'
decision — otherwise LanceDB keeps stale rows for the orphan path
until something else triggers an enqueue.
"""
decisions = reconcile(
[],
state={
"a.md": _state("a.md", status="done", change_type="added"),
},
)
assert [(d.md_path, d.change_type) for d in decisions] == [("a.md", "deleted")]
def test_done_modified_row_with_missing_path_is_recovered_as_deleted() -> None:
"""Same as the added variant, but the prior cycle was a modification."""
decisions = reconcile(
[],
state={
"a.md": _state("a.md", status="done", change_type="modified"),
},
)
assert [(d.md_path, d.change_type) for d in decisions] == [("a.md", "deleted")]
def test_mixed_scenario_preserves_order() -> None:
decisions = reconcile(
[
_scan("new.md"),
_scan("changed.md", mtime=2.0),
_scan("unchanged.md", mtime=1.0),
],
state={
"changed.md": _state(
"changed.md", mtime=1.0, status="done", change_type="modified"
),
"unchanged.md": _state(
"unchanged.md", mtime=1.0, status="done", change_type="modified"
),
"gone.md": _state("gone.md", status="pending", change_type="modified"),
},
)
by_path = {d.md_path: d.change_type for d in decisions}
assert by_path == {
"new.md": "added",
"changed.md": "modified",
"gone.md": "deleted",
}
# Order: added/modified in scan order, deleted at the tail.
assert decisions[-1].md_path == "gone.md"

83
tests/unit/test_memory/test_cascade/test_registry.py Normal file
View File

@ -0,0 +1,83 @@
"""Tests for the cascade kind registry.
Verify the 5 registered kinds' globs match the right paths and reject
noise (random ``.md``, swp files, profile-style paths). ``match_kind``
must walk the registry in declared order and pick the first matching
spec.
"""
from __future__ import annotations
import pytest
from everos.memory.cascade import KIND_REGISTRY, match_kind
@pytest.mark.parametrize(
("path", "expected_kind"),
[
(
"default_app/default_project/users/u1/episodes/episode-2026-05-14.md",
"episode",
),
("claude_code/oss/users/u_jason/episodes/episode-2026-01-01.md", "episode"),
(
"default_app/default_project/users/u1/.atomic_facts/atomic_fact-2026-05-14.md",
"atomic_fact",
),
(
"default_app/default_project/users/u1/.foresights/foresight-2026-05-14.md",
"foresight",
),
(
"default_app/default_project/agents/a1/.cases/agent_case-2026-05-14.md",
"agent_case",
),
(
"default_app/default_project/agents/a1/skills/skill_contract_risk_scan/SKILL.md",
"agent_skill",
),
],
)
def test_match_kind_recognises_registered_paths(path: str, expected_kind: str) -> None:
spec = match_kind(path)
assert spec is not None
assert spec.name == expected_kind
@pytest.mark.parametrize(
"path",
[
"users/u1/profile/user.md",
"users/u1/random.md",
"users/u1/episodes/draft.txt", # wrong extension
".cache/foo.md",
"users/u1/episodes/episode-2026-05-14.md.swp", # swap file
"agents/a1/skills/skill_x/references/notes.md", # reference, not main
# Valid episode shape but MISSING the <app>/<project> prefix — must be
# rejected so a prefix-less path can never silently match (the scanner
# would otherwise find nothing while the watcher matched, a split brain).
"users/u1/episodes/episode-2026-05-14.md",
],
)
def test_match_kind_rejects_unregistered_paths(path: str) -> None:
assert match_kind(path) is None
def test_registry_has_exactly_six_kinds() -> None:
"""The registry pins the cascade surface — no silent registration."""
names = [s.name for s in KIND_REGISTRY]
assert names == [
"episode",
"atomic_fact",
"foresight",
"agent_case",
"agent_skill",
"user_profile",
]
def test_kind_spec_path_glob_reads_off_schema() -> None:
"""Path glob is owned by the frontmatter schema, not duplicated here."""
for spec in KIND_REGISTRY:
assert spec.path_glob() == spec.frontmatter_schema.path_glob()

127
tests/unit/test_memory/test_cascade/test_scanner_unit.py Normal file
View File

@ -0,0 +1,127 @@
"""Unit tests for :class:`CascadeScanner` lifecycle + ``_collect_scan_inputs``.
The reconcile-against-state flow is integration territory; this file
covers the no-real-DB-needed pieces: idempotent start/stop and the
sync-thread walker's resilience to broken files.
"""
from __future__ import annotations
import asyncio
from pathlib import Path
import pytest
from everos.core.persistence import MemoryRoot
from everos.memory.cascade.scanner import CascadeScanner, _collect_scan_inputs
async def test_double_start_is_idempotent(tmp_path: Path) -> None:
mr = MemoryRoot(tmp_path)
mr.ensure()
scanner = CascadeScanner(mr, scan_interval_seconds=60.0)
await scanner.start()
first_task = scanner._task
await scanner.start() # second start: no-op
assert scanner._task is first_task
await scanner.stop()
async def test_stop_before_start_is_noop(tmp_path: Path) -> None:
mr = MemoryRoot(tmp_path)
mr.ensure()
scanner = CascadeScanner(mr, scan_interval_seconds=60.0)
await scanner.stop() # must not raise
async def test_double_stop_is_idempotent(tmp_path: Path) -> None:
mr = MemoryRoot(tmp_path)
mr.ensure()
scanner = CascadeScanner(mr, scan_interval_seconds=60.0)
await scanner.start()
await scanner.stop()
await scanner.stop() # second stop: no-op
def test_collect_scan_inputs_skips_dangling_symlinks(tmp_path: Path) -> None:
"""A symlink whose target was deleted yields ``stat`` OSError → skipped."""
mr = MemoryRoot(tmp_path)
mr.ensure()
# Build a real .md under a registered kind path (with the <app>/<project>
# scope prefix the glob requires), then add a broken symlink next to it to
# exercise the OSError branch.
user_dir = (
tmp_path / "default_app" / "default_project" / "users" / "u1" / "episodes"
)
user_dir.mkdir(parents=True, exist_ok=True)
real = user_dir / "episode-2026-01-01.md"
real.write_text("ok")
broken = user_dir / "episode-2026-01-02.md"
target = tmp_path / "deleted-target"
target.write_text("temp")
broken.symlink_to(target)
target.unlink() # Now ``broken`` is a dangling symlink.
inputs = _collect_scan_inputs(tmp_path)
paths = {i.md_path for i in inputs}
assert real.relative_to(tmp_path).as_posix() in paths
# Dangling symlink was silently skipped.
assert broken.relative_to(tmp_path).as_posix() not in paths
def test_collect_scan_inputs_raises_on_transient_oserror(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Non-ENOENT stat errors (EMFILE / EACCES / EIO) must propagate.
Regression guard for the 2026-05-28 incident where FD exhaustion
during a scan made every healthy md look "deleted" to reconcile().
The fix in ``_collect_scan_inputs`` swallows only ``FileNotFoundError``
and re-raises any other ``OSError`` so the reconciler never sees a
partial scan.
"""
user_dir = (
tmp_path / "default_app" / "default_project" / "users" / "u1" / "episodes"
)
user_dir.mkdir(parents=True, exist_ok=True)
real = user_dir / "episode-2026-01-01.md"
real.write_text("ok")
real_stat = Path.stat
def boom_stat(self: Path, *args, **kwargs): # type: ignore[no-untyped-def]
# Only fail on the .md file — let glob / directory walks succeed.
if self.suffix == ".md":
raise OSError(24, "Too many open files")
return real_stat(self, *args, **kwargs)
monkeypatch.setattr(Path, "stat", boom_stat)
with pytest.raises(OSError) as exc_info:
_collect_scan_inputs(tmp_path)
# errno 24 = EMFILE on every POSIX system we care about.
assert exc_info.value.errno == 24
async def test_run_loop_swallows_scan_exception(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A failure in ``scan_once`` is logged but the loop keeps going."""
mr = MemoryRoot(tmp_path)
mr.ensure()
scanner = CascadeScanner(mr, scan_interval_seconds=0.05)
call_count = {"n": 0}
async def fake_scan() -> list: # type: ignore[type-arg]
call_count["n"] += 1
if call_count["n"] == 1:
raise RuntimeError("simulated scanner failure")
return []
monkeypatch.setattr(scanner, "scan_once", fake_scan)
await scanner.start()
# Let the loop iterate at least twice (interval is 50ms).
await asyncio.sleep(0.2)
await scanner.stop()
assert call_count["n"] >= 2 # second call ran despite first throwing

36
tests/unit/test_memory/test_cascade/test_watcher_helpers.py Normal file
View File

@ -0,0 +1,36 @@
"""Unit tests for the pure helpers in :mod:`everos.memory.cascade.watcher`.
The :class:`CascadeWatcher` itself needs a running event loop + real
filesystem to test end-to-end (see ``tests/integration/``). The pure
helpers can be exercised in isolation.
"""
from __future__ import annotations
from pathlib import Path
from everos.memory.cascade.watcher import _relative_to_root, _safe_mtime
def test_relative_to_root_within(tmp_path: Path) -> None:
target = tmp_path / "users" / "u1" / "x.md"
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text("x")
assert _relative_to_root(tmp_path, str(target)) == "users/u1/x.md"
def test_relative_to_root_outside(tmp_path: Path) -> None:
"""A path outside the memory root returns ``None``."""
outside = tmp_path.parent / "completely-different" / "y.md"
assert _relative_to_root(tmp_path, str(outside)) is None
def test_safe_mtime_missing_path_returns_zero(tmp_path: Path) -> None:
missing = tmp_path / "does-not-exist.md"
assert _safe_mtime(str(missing)) == 0.0
def test_safe_mtime_existing_path_returns_positive(tmp_path: Path) -> None:
f = tmp_path / "f.md"
f.write_text("ok")
assert _safe_mtime(str(f)) > 0

573
tests/unit/test_memory/test_cascade/test_worker.py Normal file
View File

@ -0,0 +1,573 @@
"""Tests for :class:`CascadeWorker` retry classification + optimize scheduler.
The pure-function pieces (registry / reconciler) get coverage in
their own files. Here we focus on the worker's branch behaviour
without touching the real handler / lancedb stack:
- ``RecoverableError`` retries up to ``max_retry`` and then marks
``retryable=TRUE``.
- Any other exception marks ``retryable=FALSE`` immediately.
- Successful handler ⇒ ``mark_done``.
- Unknown kind ⇒ ``mark_failed(retryable=False)``.
A second group covers the per-kind throttle + trailing-edge
optimize scheduler that fires LanceDB ``optimize()`` outside the
drain loop — coalescing under burst writes, re-running when dirty
is re-raised mid-optimize, and flushing on drain-until-empty / stop.
The repo singleton is monkey-patched onto a recording fake so the
test stays in-memory.
"""
from __future__ import annotations
import asyncio
import datetime as dt
import time
import unittest.mock as mock
from dataclasses import dataclass
import pytest
from everos.memory.cascade.errors import RecoverableError, UnrecoverableError
from everos.memory.cascade.handlers import Handler, HandlerDeps
from everos.memory.cascade.types import HandlerOutcome
from everos.memory.cascade.worker import CascadeWorker
@dataclass
class _Row:
"""Minimal MdChangeState shape the worker reads off."""
md_path: str
kind: str = "episode"
change_type: str = "added"
retry_count: int = 0
class _FakeRepo:
"""Records every state-machine transition the worker drives."""
def __init__(self, batch: list[_Row]) -> None:
self.batch = list(batch)
self.done: list[str] = []
self.failed: list[tuple[str, bool, str, int]] = []
async def claim_pending_batch(self, _limit: int) -> list[_Row]:
items, self.batch = self.batch, []
return items
async def mark_done(self, md_path: str) -> None:
self.done.append(md_path)
async def mark_failed(
self,
md_path: str,
*,
retryable: bool,
error: str,
new_retry_count: int,
) -> None:
self.failed.append((md_path, retryable, error, new_retry_count))
class _OkHandler(Handler):
def __init__(self) -> None: # noqa: D401 — no deps needed
pass
async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome:
return HandlerOutcome(
md_path=md_path, kind="episode", upserted=1, deleted=0, skipped=0
)
async def handle_deleted(self, md_path: str) -> HandlerOutcome:
return HandlerOutcome(
md_path=md_path, kind="episode", upserted=0, deleted=1, skipped=0
)
class _RecoverableHandler(_OkHandler):
"""Always raises RecoverableError."""
async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome:
raise RecoverableError("embedding 503")
class _UnrecoverableHandler(_OkHandler):
async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome:
raise UnrecoverableError("YAML parse error")
class _BareExceptionHandler(_OkHandler):
async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome:
raise RuntimeError("unexpected boom")
@pytest.fixture
def patched_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeRepo:
"""Drop a fake repo onto the module the worker imports."""
from everos.memory.cascade import worker as worker_mod
repo = _FakeRepo(batch=[])
monkeypatch.setattr(worker_mod, "md_change_state_repo", repo)
return repo
async def test_ok_handler_marks_done(patched_repo: _FakeRepo) -> None:
patched_repo.batch = [_Row(md_path="a.md")]
w = CascadeWorker({"episode": _OkHandler()}, retry_backoff_seconds=0)
await w.drain_once()
assert patched_repo.done == ["a.md"]
assert patched_repo.failed == []
async def test_recoverable_handler_marks_retryable_after_max_retry(
patched_repo: _FakeRepo,
) -> None:
patched_repo.batch = [_Row(md_path="a.md")]
w = CascadeWorker(
{"episode": _RecoverableHandler()}, max_retry=2, retry_backoff_seconds=0
)
await w.drain_once()
assert patched_repo.done == []
assert len(patched_repo.failed) == 1
path, retryable, _err, retry_count = patched_repo.failed[0]
assert path == "a.md"
assert retryable is True
assert retry_count == 2 # 2 retries after the initial attempt
async def test_unrecoverable_handler_marks_permanent(
patched_repo: _FakeRepo,
) -> None:
patched_repo.batch = [_Row(md_path="a.md")]
w = CascadeWorker({"episode": _UnrecoverableHandler()}, retry_backoff_seconds=0)
await w.drain_once()
_path, retryable, err, _retry = patched_repo.failed[0]
assert retryable is False
assert "UnrecoverableError" in err or "YAML parse error" in err
async def test_bare_exception_marked_permanent(patched_repo: _FakeRepo) -> None:
"""Anything that isn't RecoverableError counts as unrecoverable."""
patched_repo.batch = [_Row(md_path="a.md")]
w = CascadeWorker({"episode": _BareExceptionHandler()}, retry_backoff_seconds=0)
await w.drain_once()
_path, retryable, _err, _retry = patched_repo.failed[0]
assert retryable is False
async def test_unknown_kind_marks_permanent_without_handler(
patched_repo: _FakeRepo,
) -> None:
patched_repo.batch = [_Row(md_path="a.md", kind="mystery")]
w = CascadeWorker({"episode": _OkHandler()}, retry_backoff_seconds=0)
await w.drain_once()
assert patched_repo.failed[0][1] is False
assert "no handler" in patched_repo.failed[0][2]
async def test_drain_until_empty_loops_until_no_batch(
patched_repo: _FakeRepo,
) -> None:
"""Worker keeps draining until claim returns an empty list."""
rows = [_Row(md_path=f"a{i}.md") for i in range(3)]
class _ChunkedRepo(_FakeRepo):
async def claim_pending_batch(self, _limit: int) -> list[_Row]:
if not self.batch:
return []
head, self.batch = self.batch[:1], self.batch[1:]
return head
chunked = _ChunkedRepo(rows)
from everos.memory.cascade import worker as worker_mod
with mock.patch.object(worker_mod, "md_change_state_repo", chunked):
w = CascadeWorker({"episode": _OkHandler()}, retry_backoff_seconds=0)
total = await w.drain_until_empty()
assert total == 3
assert len(chunked.done) == 3
def test_worker_handler_deps_construct_with_real_classes() -> None:
"""Sanity: HandlerDeps accepts the real provider Protocols."""
# No instantiation needed — just verifies the dataclass shape.
assert {"memory_root", "embedder", "tokenizer"} == {
f.name for f in HandlerDeps.__dataclass_fields__.values()
}
# ── Optimize scheduler tests ───────────────────────────────────────────────
class _FakeLanceRepo:
"""Records every optimize() / rebuild_indexes() call.
``optimize_delay`` / ``rebuild_delay`` simulate slow operations.
``rebuild_raises`` makes ``rebuild_indexes`` raise (for crash-safety tests).
Each ``optimize`` call's ``cleanup_older_than`` is preserved so
prune-cadence tests can assert which calls took the heavy path.
"""
def __init__(
self,
*,
optimize_delay: float = 0.0,
rebuild_delay: float = 0.0,
rebuild_raises: bool = False,
) -> None:
self.optimize_calls: list[float] = []
self.optimize_cleanup_args: list[dt.timedelta | None] = []
self.rebuild_calls: list[float] = []
self.optimize_delay = optimize_delay
self.rebuild_delay = rebuild_delay
self.rebuild_raises = rebuild_raises
async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None:
if self.optimize_delay > 0:
await asyncio.sleep(self.optimize_delay)
self.optimize_calls.append(time.monotonic())
self.optimize_cleanup_args.append(cleanup_older_than)
async def rebuild_indexes(self) -> None:
if self.rebuild_delay > 0:
await asyncio.sleep(self.rebuild_delay)
if self.rebuild_raises:
raise RuntimeError("rebuild boom")
self.rebuild_calls.append(time.monotonic())
class _OkHandlerWithRepo(_OkHandler):
"""OK handler exposing a fake ``lance_repo`` for scheduler tests."""
def __init__(self, repo: _FakeLanceRepo) -> None:
super().__init__()
self.lance_repo = repo
async def test_schedule_optimize_noop_when_handler_has_no_lance_repo(
patched_repo: _FakeRepo,
) -> None:
"""Test stubs without ``lance_repo`` should not even register state."""
w = CascadeWorker(
{"episode": _OkHandler()},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.05,
)
w._schedule_optimize("episode")
assert "episode" not in w._optimizer_states
async def test_schedule_optimize_collapses_burst_within_throttle_window(
patched_repo: _FakeRepo,
) -> None:
"""A burst of synchronous schedules creates at most one in-flight task.
The first call starts the optimize; subsequent calls during the
same window only flip ``dirty``. With no time advance between
schedules, the runner sees ``dirty=False`` after the first run
and exits — total optimize() calls collapse to one.
"""
fake = _FakeLanceRepo()
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.05,
)
for _ in range(10):
w._schedule_optimize("episode")
await w._flush_optimizers()
assert fake.optimize_calls, "expected at least one optimize"
assert len(fake.optimize_calls) == 1, (
f"burst should collapse, got {len(fake.optimize_calls)} calls"
)
async def test_schedule_optimize_reruns_when_dirty_set_during_optimize(
patched_repo: _FakeRepo,
) -> None:
"""A write that lands mid-optimize re-raises ``dirty`` and triggers a re-run.
Uses an artificially slow optimize so the second schedule fires
while the first run is still in flight. Trailing-edge semantics
guarantee the second run happens after the throttle interval.
"""
fake = _FakeLanceRepo(optimize_delay=0.05)
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.02,
)
w._schedule_optimize("episode")
await asyncio.sleep(0.01) # ensure first task is mid-optimize
w._schedule_optimize("episode")
await w._flush_optimizers()
assert len(fake.optimize_calls) == 2
async def test_concurrent_schedules_keep_one_task_per_kind(
patched_repo: _FakeRepo,
) -> None:
"""LanceDB manifest contention guard: per-kind in-flight task is unique."""
fake = _FakeLanceRepo(optimize_delay=0.05)
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.02,
)
w._schedule_optimize("episode")
first_task = w._optimizer_states["episode"].task
# Re-schedule while first task is still in flight; slot must not
# be replaced.
for _ in range(5):
w._schedule_optimize("episode")
assert w._optimizer_states["episode"].task is first_task
await w._flush_optimizers()
async def test_flush_optimizers_awaits_pending_task(
patched_repo: _FakeRepo,
) -> None:
"""flush_optimizers blocks until in-flight optimize commits and clears slot."""
fake = _FakeLanceRepo(optimize_delay=0.05)
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.02,
)
w._schedule_optimize("episode")
assert w._optimizer_states["episode"].task is not None
await w._flush_optimizers()
assert fake.optimize_calls, "flush should not return before optimize ran"
assert w._optimizer_states["episode"].task is None
async def test_drain_until_empty_flushes_optimizers_before_returning(
patched_repo: _FakeRepo,
) -> None:
"""CLI ``cascade sync`` expects FTS to be current when the call returns."""
fake = _FakeLanceRepo(optimize_delay=0.03)
patched_repo.batch = [_Row(md_path="a.md")]
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.02,
)
await w.drain_until_empty()
assert patched_repo.done == ["a.md"]
assert len(fake.optimize_calls) == 1
assert w._optimizer_states["episode"].task is None
async def test_drain_once_does_not_block_on_optimize(
patched_repo: _FakeRepo,
) -> None:
"""drain_once is fire-and-forget — it must return before optimize commits."""
fake = _FakeLanceRepo(optimize_delay=0.2)
patched_repo.batch = [_Row(md_path="a.md")]
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.01,
)
started = time.monotonic()
await w.drain_once()
drain_elapsed = time.monotonic() - started
# drain returned long before the 0.2s optimize would finish
assert drain_elapsed < 0.1, f"drain blocked on optimize: {drain_elapsed:.3f}s"
assert not fake.optimize_calls, "optimize should still be in flight"
await w._flush_optimizers()
assert len(fake.optimize_calls) == 1
async def test_stop_waits_for_in_flight_optimize(
patched_repo: _FakeRepo,
) -> None:
"""stop() must give an in-flight optimize a chance to commit cleanly."""
fake = _FakeLanceRepo(optimize_delay=0.05)
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.02,
optimize_heartbeat_seconds=10.0,
# Park rebuild interval — startup sweep still fires but we wait
# for it before testing optimize semantics.
optimize_rebuild_interval_seconds=10.0,
)
await w.start()
# Let the startup rebuild sweep complete (instant for the fake repo)
# before scheduling optimize — otherwise optimize would queue behind it.
await asyncio.sleep(0.02)
assert fake.rebuild_calls, "startup rebuild should have fired by now"
w._schedule_optimize("episode")
await asyncio.sleep(0.01) # let optimize start
await w.stop()
assert len(fake.optimize_calls) == 1
async def test_optimize_failure_does_not_crash_drain_loop(
patched_repo: _FakeRepo,
) -> None:
"""Repo.optimize() raising should be logged but never propagate."""
class _FailingRepo:
async def optimize(self) -> None:
raise RuntimeError("simulated lancedb manifest conflict")
class _HandlerWithFailingRepo(_OkHandler):
def __init__(self) -> None:
super().__init__()
self.lance_repo = _FailingRepo()
patched_repo.batch = [_Row(md_path="a.md")]
w = CascadeWorker(
{"episode": _HandlerWithFailingRepo()},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.02,
)
# If the failure propagated, drain_until_empty would raise.
await w.drain_until_empty()
assert patched_repo.done == ["a.md"]
assert patched_repo.failed == []
async def test_heartbeat_schedules_every_handler_kind(
patched_repo: _FakeRepo,
) -> None:
"""The heartbeat sweeps all kinds, even ones nobody wrote to.
Drives the heartbeat manually via a short interval and asserts
that ``optimize`` ran for both kinds at least once.
"""
fake_a = _FakeLanceRepo()
fake_b = _FakeLanceRepo()
w = CascadeWorker(
{
"episode": _OkHandlerWithRepo(fake_a),
"atomic_fact": _OkHandlerWithRepo(fake_b),
},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.01,
optimize_heartbeat_seconds=0.05,
)
await w.start()
# Let at least one heartbeat tick happen.
await asyncio.sleep(0.12)
await w.stop()
assert fake_a.optimize_calls, "heartbeat should have scheduled episode"
assert fake_b.optimize_calls, "heartbeat should have scheduled atomic_fact"
async def test_optimize_prunes_on_first_call_then_throttles(
patched_repo: _FakeRepo,
) -> None:
"""First optimize() per kind passes ``cleanup_older_than``; subsequent
calls within ``optimize_prune_interval_seconds`` do not.
Rationale lives in ``DEFAULT_OPTIMIZE_PRUNE_INTERVAL_SECONDS``:
LanceDB ``optimize()`` without ``cleanup_older_than`` leaves stale
physical files on disk; passing it on every 1-second optimize tick
is wasteful, but never passing it leaks files until FDs exhaust.
A separate cadence — prune ≪ optimize — balances the two.
"""
fake = _FakeLanceRepo()
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.01,
optimize_prune_interval_seconds=10.0, # long — second call should NOT prune
)
# First call: state has never pruned, must include cleanup_older_than.
w._schedule_optimize("episode")
await w._flush_optimizers()
assert len(fake.optimize_calls) == 1
assert fake.optimize_cleanup_args[0] is not None, (
"first optimize must prune to catch up from prior session"
)
assert fake.optimize_cleanup_args[0] == dt.timedelta(seconds=10.0)
# Second call within the prune window: light path (no cleanup).
await asyncio.sleep(0.02) # exceed optimize throttle (0.01), not prune (10)
w._schedule_optimize("episode")
await w._flush_optimizers()
assert len(fake.optimize_calls) == 2
assert fake.optimize_cleanup_args[1] is None, (
"second optimize within prune window should skip cleanup_older_than"
)
# ── Rebuild scheduler tests ────────────────────────────────────────────────
async def test_rebuild_runs_on_startup_for_every_kind(
patched_repo: _FakeRepo,
) -> None:
"""The first rebuild sweep fires on worker start, before any interval.
Otherwise a daemon that restarts more often than the rebuild
interval would never bound accumulated UUIDs.
"""
fake_a = _FakeLanceRepo()
fake_b = _FakeLanceRepo()
w = CascadeWorker(
{
"episode": _OkHandlerWithRepo(fake_a),
"atomic_fact": _OkHandlerWithRepo(fake_b),
},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.01,
optimize_heartbeat_seconds=10.0, # park heartbeat
optimize_rebuild_interval_seconds=10.0, # only the startup sweep should fire
)
await w.start()
# Allow the startup sweep to complete; the next tick is 10s away.
await asyncio.sleep(0.1)
await w.stop()
# Exactly one rebuild per kind: the startup sweep. Next interval is 10s.
assert len(fake_a.rebuild_calls) == 1
assert len(fake_b.rebuild_calls) == 1
async def test_rebuild_runs_periodically(
patched_repo: _FakeRepo,
) -> None:
"""After the startup sweep, rebuild repeats every interval."""
fake = _FakeLanceRepo()
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.01,
optimize_heartbeat_seconds=10.0,
optimize_rebuild_interval_seconds=0.05, # ~tick every 50ms in this test
)
await w.start()
await asyncio.sleep(0.2) # ~4 ticks plus startup sweep
await w.stop()
# Startup sweep + at least 2 interval-driven sweeps.
assert len(fake.rebuild_calls) >= 3, (
f"expected ≥3 rebuilds (1 startup + ≥2 periodic), got {len(fake.rebuild_calls)}"
)
async def test_rebuild_failure_does_not_crash_daemon(
patched_repo: _FakeRepo,
) -> None:
"""A throwing rebuild is logged and absorbed; the worker keeps running."""
fake = _FakeLanceRepo(rebuild_raises=True)
w = CascadeWorker(
{"episode": _OkHandlerWithRepo(fake)},
retry_backoff_seconds=0,
optimize_min_interval_seconds=0.01,
optimize_heartbeat_seconds=0.05,
optimize_rebuild_interval_seconds=10.0,
)
await w.start()
# Give startup rebuild a chance to throw, then heartbeat to keep optimizing.
await asyncio.sleep(0.12)
# Optimize should still progress despite rebuild errors.
assert fake.optimize_calls, "heartbeat optimize should run even when rebuild fails"
await w.stop()
# Worker is still alive (stop() returned cleanly).
assert w._task is None