chore: initialize EverOS 1.0.0
md-first memory extraction framework for AI agents. Markdown is the single source of truth; SQLite holds state and LanceDB provides the rebuildable vector + BM25 + scalar index. The codebase follows a single-direction DDD layering (entrypoints -> service -> memory -> infra, with component / core / config cross-cutting) enforced by import-linter. Engineering surface: - Coding conventions in .claude/rules/ (path-scoped) and workflows in .claude/skills/ (/commit, /new-branch, /pr). - GitHub Actions CI runs make lint + test + integration; pre-commit mirrors the gates locally (ruff, hygiene hooks, gitlint commit-msg). - Commit messages follow Conventional Commits, enforced by gitlint. - make lint also enforces datetime two-zone discipline and OpenAPI drift.
This commit is contained in:
Elliot Chen
1
src/everos/infra/ome/_stores/__init__.py
Normal file
1
src/everos/infra/ome/_stores/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
"""Internal: SQLite-backed state stores (counter / idle / run_record)."""
|
||||
107
src/everos/infra/ome/_stores/counter.py
Normal file
107
src/everos/infra/ome/_stores/counter.py
Normal file
@ -0,0 +1,107 @@
|
||||
"""CounterStore — persistent (strategy_name, bucket_key) → counter rows.
|
||||
|
||||
Backs the ``Counter`` gate in OME's dispatch pipeline: each call to
|
||||
:meth:`CounterStore.incr_and_check` atomically increments the bucket's
|
||||
counter and reports whether the strategy should fire this time.
|
||||
|
||||
Pass semantics:
|
||||
- ``counter >= threshold`` AND cooldown elapsed → ``passed=True``
|
||||
- On pass, the row's counter resets to 0 and ``last_passed_ts``
|
||||
advances to ``now``; the next pass needs a fresh accumulation.
|
||||
- ``cooldown_seconds=0`` disables the cooldown gate (threshold alone).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
from everos.component.utils.datetime import (
|
||||
from_iso_format,
|
||||
get_utc_now,
|
||||
to_iso_format,
|
||||
)
|
||||
from everos.infra.ome._stores.storage import OMEStorage
|
||||
|
||||
|
||||
class CounterStore:
|
||||
"""SQLite-backed counter for the ``Counter`` gate (see module docstring)."""
|
||||
|
||||
def __init__(self, storage: OMEStorage) -> None:
|
||||
self._storage = storage
|
||||
|
||||
async def incr_and_check(
|
||||
self,
|
||||
strategy_name: str,
|
||||
bucket_key: str,
|
||||
*,
|
||||
threshold: int,
|
||||
cooldown_seconds: int,
|
||||
) -> tuple[bool, int]:
|
||||
"""Increment ``(strategy_name, bucket_key)``'s counter atomically.
|
||||
|
||||
Args:
|
||||
strategy_name: Strategy whose counter to update.
|
||||
bucket_key: The bucket value derived from the event field
|
||||
(or ``"__all__"`` when the gate is unbucketed).
|
||||
threshold: Pass once the counter reaches this value
|
||||
(``>=``).
|
||||
cooldown_seconds: Minimum seconds since the last pass for
|
||||
the strategy/bucket; ``0`` disables the cooldown check.
|
||||
|
||||
Returns:
|
||||
``(passed, counter)``. ``counter`` is the counter value at
|
||||
the moment of the check (i.e. pre-reset on pass). Useful for
|
||||
diagnostics — ``threshold`` is *not* substituted, so callers
|
||||
observing ``counter > threshold`` learn the gate is
|
||||
over-armed (e.g. threshold was lowered via hot reload while
|
||||
the counter had already accumulated past the new value).
|
||||
"""
|
||||
now = get_utc_now()
|
||||
async with self._storage.transaction() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT counter, last_passed_ts FROM counter_store "
|
||||
"WHERE strategy_name = ? AND bucket_key = ?",
|
||||
(strategy_name, bucket_key),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
counter = (row[0] if row else 0) + 1
|
||||
last_passed = from_iso_format(row[1]) if row and row[1] else None
|
||||
|
||||
cooldown_ok = (
|
||||
cooldown_seconds == 0
|
||||
or last_passed is None
|
||||
or now - last_passed >= timedelta(seconds=cooldown_seconds)
|
||||
)
|
||||
passed = counter >= threshold and cooldown_ok
|
||||
|
||||
new_counter = 0 if passed else counter
|
||||
new_last_passed_ts = (
|
||||
to_iso_format(now)
|
||||
if passed
|
||||
else (to_iso_format(last_passed) if last_passed else None)
|
||||
)
|
||||
await conn.execute(
|
||||
"INSERT INTO counter_store (strategy_name, bucket_key, "
|
||||
"counter, last_passed_ts) "
|
||||
"VALUES (?, ?, ?, ?) "
|
||||
"ON CONFLICT(strategy_name, bucket_key) DO UPDATE SET "
|
||||
"counter = excluded.counter, "
|
||||
"last_passed_ts = excluded.last_passed_ts",
|
||||
(strategy_name, bucket_key, new_counter, new_last_passed_ts),
|
||||
)
|
||||
return passed, counter
|
||||
|
||||
async def get_progress(self, strategy_name: str, bucket_key: str) -> int:
|
||||
"""Return the counter value persisted for this bucket (0 if absent).
|
||||
|
||||
Read-only; does not increment. Used by dispatcher inspect-mode
|
||||
to report progress without mutating state.
|
||||
"""
|
||||
async with self._storage.connect() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT counter FROM counter_store "
|
||||
"WHERE strategy_name = ? AND bucket_key = ?",
|
||||
(strategy_name, bucket_key),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return row[0] if row else 0
|
||||
64
src/everos/infra/ome/_stores/idle.py
Normal file
64
src/everos/infra/ome/_stores/idle.py
Normal file
@ -0,0 +1,64 @@
|
||||
"""IdleStore — last_activity_ts rows backing the Idle trigger.
|
||||
|
||||
All writes pass through ``to_iso_format`` over a tz-aware datetime, so
|
||||
``last_activity_ts`` is a fixed-format ISO 8601 string whose
|
||||
lexicographic order matches temporal order — :meth:`scan_idle` relies
|
||||
on this to keep the column un-wrapped in its predicate so SQLite can
|
||||
use ``idx_idle_scan``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from everos.component.utils.datetime import from_iso_format, to_iso_format
|
||||
from everos.infra.ome._stores.storage import OMEStorage
|
||||
|
||||
|
||||
class IdleStore:
|
||||
"""SQLite-backed last-activity tracker for the ``Idle`` trigger."""
|
||||
|
||||
def __init__(self, storage: OMEStorage) -> None:
|
||||
self._storage = storage
|
||||
|
||||
async def touch(self, strategy_name: str, bucket_key: str, *, at: datetime) -> None:
|
||||
"""UPSERT ``last_activity_ts = at`` for ``(strategy_name, bucket_key)``."""
|
||||
async with self._storage.connect() as conn:
|
||||
await conn.execute(
|
||||
"INSERT INTO idle_store "
|
||||
"(strategy_name, bucket_key, last_activity_ts) "
|
||||
"VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(strategy_name, bucket_key) DO UPDATE SET "
|
||||
"last_activity_ts = excluded.last_activity_ts",
|
||||
(strategy_name, bucket_key, to_iso_format(at)),
|
||||
)
|
||||
await conn.commit()
|
||||
|
||||
async def scan_idle(
|
||||
self, strategy_name: str, *, idle_seconds: int, now: datetime
|
||||
) -> list[str]:
|
||||
"""Return bucket_keys with ``last_activity_ts`` older than ``idle_seconds``."""
|
||||
# Cutoff on the RHS so the indexed column stays un-wrapped.
|
||||
cutoff = to_iso_format(now - timedelta(seconds=idle_seconds))
|
||||
async with self._storage.connect() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT bucket_key FROM idle_store "
|
||||
"WHERE strategy_name = ? AND last_activity_ts <= ? "
|
||||
"ORDER BY last_activity_ts ASC",
|
||||
(strategy_name, cutoff),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
async def get_last_activity(
|
||||
self, strategy_name: str, bucket_key: str
|
||||
) -> datetime | None:
|
||||
"""Return the stored ``last_activity_ts`` (``None`` if never touched)."""
|
||||
async with self._storage.connect() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT last_activity_ts FROM idle_store "
|
||||
"WHERE strategy_name = ? AND bucket_key = ?",
|
||||
(strategy_name, bucket_key),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return from_iso_format(row[0]) if row else None
|
||||
168
src/everos/infra/ome/_stores/run_record.py
Normal file
168
src/everos/infra/ome/_stores/run_record.py
Normal file
@ -0,0 +1,168 @@
|
||||
"""RunRecord persistence — state machine writes + same-transaction ring-buffer trim.
|
||||
|
||||
State machine (one row per ``run_id``):
|
||||
RUNNING → SUCCESS / FAILED / DEAD_LETTER / CRASHED
|
||||
|
||||
Every :meth:`RunRecordStore.mark_running` INSERT runs inside one
|
||||
``BEGIN IMMEDIATE`` transaction with a paired DELETE that keeps only
|
||||
the newest ``max_records_per_strategy`` rows for that strategy. Bound
|
||||
is enforced atomically — no background sweeper, no transient
|
||||
over-budget state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from everos.component.utils.datetime import (
|
||||
from_iso_format,
|
||||
get_utc_now,
|
||||
to_iso_format,
|
||||
)
|
||||
from everos.infra.ome._stores.storage import OMEStorage
|
||||
from everos.infra.ome.records import RunRecord, RunStatus
|
||||
|
||||
|
||||
class RunRecordStore:
|
||||
"""SQLite-backed persistence for ``RunRecord`` (see module docstring)."""
|
||||
|
||||
def __init__(self, storage: OMEStorage, max_records_per_strategy: int) -> None:
|
||||
self._storage = storage
|
||||
self._max = max_records_per_strategy
|
||||
|
||||
async def mark_running(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
strategy_name: str,
|
||||
attempt: int,
|
||||
event_topic: str,
|
||||
event_payload: str,
|
||||
max_retries_snapshot: int,
|
||||
) -> None:
|
||||
"""Insert a new RUNNING row and trim the strategy's ring buffer atomically."""
|
||||
async with self._storage.transaction() as conn:
|
||||
await conn.execute(
|
||||
"INSERT INTO run_record "
|
||||
"(run_id, strategy_name, status, attempt, started_at, "
|
||||
" event_topic, event_payload, max_retries_snapshot) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
run_id,
|
||||
strategy_name,
|
||||
RunStatus.RUNNING.value,
|
||||
attempt,
|
||||
to_iso_format(get_utc_now()),
|
||||
event_topic,
|
||||
event_payload,
|
||||
max_retries_snapshot,
|
||||
),
|
||||
)
|
||||
await conn.execute(
|
||||
"DELETE FROM run_record "
|
||||
"WHERE strategy_name = ? AND run_id NOT IN ("
|
||||
" SELECT run_id FROM run_record WHERE strategy_name = ? "
|
||||
" ORDER BY started_at DESC LIMIT ?)",
|
||||
(strategy_name, strategy_name, self._max),
|
||||
)
|
||||
|
||||
async def mark_success(self, *, run_id: str, finished_at: datetime) -> None:
|
||||
"""Mark RUNNING → SUCCESS."""
|
||||
await self._update_status(run_id, RunStatus.SUCCESS, finished_at, None)
|
||||
|
||||
async def mark_failed(
|
||||
self, *, run_id: str, finished_at: datetime, error: str
|
||||
) -> None:
|
||||
"""Mark RUNNING → FAILED (retry pending)."""
|
||||
await self._update_status(run_id, RunStatus.FAILED, finished_at, error)
|
||||
|
||||
async def mark_dead_letter(
|
||||
self, *, run_id: str, finished_at: datetime, error: str
|
||||
) -> None:
|
||||
"""Mark RUNNING → DEAD_LETTER (retries exhausted or non-retryable)."""
|
||||
await self._update_status(run_id, RunStatus.DEAD_LETTER, finished_at, error)
|
||||
|
||||
async def mark_crashed(
|
||||
self, *, run_id: str, finished_at: datetime, error: str
|
||||
) -> None:
|
||||
"""Mark RUNNING → CRASHED (called by crash-recovery sweep)."""
|
||||
await self._update_status(run_id, RunStatus.CRASHED, finished_at, error)
|
||||
|
||||
async def _update_status(
|
||||
self,
|
||||
run_id: str,
|
||||
status: RunStatus,
|
||||
finished_at: datetime,
|
||||
error: str | None,
|
||||
) -> None:
|
||||
async with self._storage.connect() as conn:
|
||||
await conn.execute(
|
||||
"UPDATE run_record "
|
||||
"SET status = ?, finished_at = ?, error = ? "
|
||||
"WHERE run_id = ?",
|
||||
(status.value, to_iso_format(finished_at), error, run_id),
|
||||
)
|
||||
await conn.commit()
|
||||
|
||||
async def get(self, run_id: str) -> RunRecord | None:
|
||||
"""Return the record for ``run_id`` (``None`` if absent)."""
|
||||
async with self._storage.connect() as conn:
|
||||
cur = await conn.execute(
|
||||
_SELECT_COLUMNS + " WHERE run_id = ?",
|
||||
(run_id,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _row_to_record(row) if row else None
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
strategy_name: str,
|
||||
status: RunStatus | None = None,
|
||||
limit: int = 100,
|
||||
) -> list[RunRecord]:
|
||||
"""Return ``strategy_name``'s records, newest first; optional status filter."""
|
||||
sql = _SELECT_COLUMNS + " WHERE strategy_name = ?"
|
||||
args: list[Any] = [strategy_name]
|
||||
if status is not None:
|
||||
sql += " AND status = ?"
|
||||
args.append(status.value)
|
||||
sql += " ORDER BY started_at DESC LIMIT ?"
|
||||
args.append(limit)
|
||||
async with self._storage.connect() as conn:
|
||||
cur = await conn.execute(sql, args)
|
||||
rows = await cur.fetchall()
|
||||
return [_row_to_record(r) for r in rows]
|
||||
|
||||
async def find_running(self) -> list[RunRecord]:
|
||||
"""Return every row still in RUNNING — used by crash recovery at start()."""
|
||||
async with self._storage.connect() as conn:
|
||||
cur = await conn.execute(
|
||||
_SELECT_COLUMNS + " WHERE status = ?",
|
||||
(RunStatus.RUNNING.value,),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [_row_to_record(r) for r in rows]
|
||||
|
||||
|
||||
_SELECT_COLUMNS = (
|
||||
"SELECT run_id, strategy_name, status, attempt, started_at, finished_at, "
|
||||
" error, event_topic, event_payload, max_retries_snapshot "
|
||||
"FROM run_record"
|
||||
)
|
||||
|
||||
|
||||
def _row_to_record(row: tuple) -> RunRecord:
|
||||
return RunRecord(
|
||||
run_id=row[0],
|
||||
strategy_name=row[1],
|
||||
status=RunStatus(row[2]),
|
||||
attempt=row[3],
|
||||
started_at=from_iso_format(row[4]),
|
||||
finished_at=from_iso_format(row[5]) if row[5] else None,
|
||||
error=row[6],
|
||||
event_topic=row[7],
|
||||
event_payload=row[8],
|
||||
max_retries_snapshot=row[9],
|
||||
)
|
||||
115
src/everos/infra/ome/_stores/storage.py
Normal file
115
src/everos/infra/ome/_stores/storage.py
Normal file
@ -0,0 +1,115 @@
|
||||
"""OME SQLite storage — schema initialization + connection factory.
|
||||
|
||||
Single file (default ``MemoryRoot.default().ome_db`` ≡
|
||||
``<memory-root>/.index/sqlite/ome.db``). Holds 3 OME-managed tables
|
||||
(counter_store / idle_store / run_record); APS jobstore table is created
|
||||
by APScheduler itself when its SQLAlchemyJobStore connects.
|
||||
|
||||
PRAGMA scopes (see https://www.sqlite.org/pragma.html):
|
||||
- ``journal_mode=WAL`` is file-level — persisted in the db header,
|
||||
applied once in :meth:`OMEStorage.init`.
|
||||
- ``synchronous=NORMAL``, ``cache_size=-65536``, ``busy_timeout=5000``
|
||||
are connection-level and reset on every new connection, so they are
|
||||
re-applied inside :meth:`OMEStorage.connect` (which is why
|
||||
``connect`` is an ``@asynccontextmanager`` rather than a passthrough).
|
||||
This mirrors SQLAlchemy's canonical ``@event.listens_for(Engine,
|
||||
"connect")`` pattern for SQLite — aiosqlite exposes no equivalent
|
||||
hook. ``busy_timeout=5000`` matters because the APS jobstore writes
|
||||
its own table in the same db file; without it, WAL writer-vs-writer
|
||||
contention surfaces as ``SQLITE_BUSY`` instead of brief backoff.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import aiosqlite
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS counter_store (
|
||||
strategy_name TEXT NOT NULL,
|
||||
bucket_key TEXT NOT NULL,
|
||||
counter INTEGER NOT NULL DEFAULT 0,
|
||||
last_passed_ts TIMESTAMP,
|
||||
PRIMARY KEY (strategy_name, bucket_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS idle_store (
|
||||
strategy_name TEXT NOT NULL,
|
||||
bucket_key TEXT NOT NULL,
|
||||
last_activity_ts TIMESTAMP NOT NULL,
|
||||
PRIMARY KEY (strategy_name, bucket_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_idle_scan
|
||||
ON idle_store (strategy_name, last_activity_ts);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS run_record (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
strategy_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 0,
|
||||
started_at TIMESTAMP NOT NULL,
|
||||
finished_at TIMESTAMP,
|
||||
error TEXT,
|
||||
event_topic TEXT NOT NULL,
|
||||
event_payload TEXT NOT NULL,
|
||||
max_retries_snapshot INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_run_strategy_started
|
||||
ON run_record (strategy_name, started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_run_status_started
|
||||
ON run_record (status, started_at DESC);
|
||||
"""
|
||||
|
||||
_INIT_PRAGMAS = ("PRAGMA journal_mode=WAL",)
|
||||
_CONN_PRAGMAS = (
|
||||
"PRAGMA synchronous=NORMAL",
|
||||
"PRAGMA cache_size=-65536",
|
||||
"PRAGMA busy_timeout=5000",
|
||||
)
|
||||
|
||||
|
||||
class OMEStorage:
|
||||
"""Connection factory + schema init for the OME SQLite db."""
|
||||
|
||||
def __init__(self, db_path: Path) -> None:
|
||||
self.db_path = db_path
|
||||
|
||||
async def init(self) -> None:
|
||||
"""Create parent dirs + apply file-level pragmas + create schema."""
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiosqlite.connect(self.db_path) as conn:
|
||||
for pragma in _INIT_PRAGMAS:
|
||||
await conn.execute(pragma)
|
||||
await conn.executescript(_SCHEMA)
|
||||
await conn.commit()
|
||||
|
||||
@asynccontextmanager
|
||||
async def connect(self) -> AsyncIterator[aiosqlite.Connection]:
|
||||
"""Yield an aiosqlite connection with per-connection pragmas applied."""
|
||||
async with aiosqlite.connect(self.db_path) as conn:
|
||||
for pragma in _CONN_PRAGMAS:
|
||||
await conn.execute(pragma)
|
||||
yield conn
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self) -> AsyncIterator[aiosqlite.Connection]:
|
||||
"""Yield a connection inside an ``IMMEDIATE`` transaction.
|
||||
|
||||
Commits on success, rolls back on any exception. Mirrors
|
||||
SQLAlchemy's ``conn.begin()`` for raw aiosqlite, which exposes
|
||||
no built-in transaction context manager. ``BEGIN IMMEDIATE``
|
||||
(rather than ``DEFERRED``) acquires the write lock upfront so
|
||||
a read-modify-write block cannot lose to a competing writer
|
||||
between its SELECT and its UPDATE.
|
||||
"""
|
||||
async with self.connect() as conn:
|
||||
try:
|
||||
await conn.execute("BEGIN IMMEDIATE")
|
||||
yield conn
|
||||
await conn.commit()
|
||||
except Exception:
|
||||
await conn.rollback()
|
||||
raise
|
||||
Reference in New Issue
Block a user