tomtan/EverOS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: initialize EverOS 1.0.0

Browse Source 
md-first memory extraction framework for AI agents.

Markdown is the single source of truth; SQLite holds state and LanceDB
provides the rebuildable vector + BM25 + scalar index. The codebase follows
a single-direction DDD layering (entrypoints -> service -> memory -> infra,
with component / core / config cross-cutting) enforced by import-linter.

Engineering surface:
- Coding conventions in .claude/rules/ (path-scoped) and workflows in
  .claude/skills/ (/commit, /new-branch, /pr).
- GitHub Actions CI runs make lint + test + integration; pre-commit mirrors
  the gates locally (ruff, hygiene hooks, gitlint commit-msg).
- Commit messages follow Conventional Commits, enforced by gitlint.
- make lint also enforces datetime two-zone discipline and OpenAPI drift.
This commit is contained in:
Elliot Chen
2026-06-05 22:35:51 +08:00
commit 518b8eca85
636 changed files with 160553 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/everos/entrypoints/cli/commands/__init__.py Normal file
View File

@ -0,0 +1,5 @@
"""CLI subcommand modules.
Each module here exposes a ``app: typer.Typer`` instance which is mounted
as a subcommand group by :mod:`everos.entrypoints.cli.main`.
"""

267
src/everos/entrypoints/cli/commands/cascade.py Normal file
View File

@ -0,0 +1,267 @@
"""``everos cascade`` subcommand group.
Three one-shot operations on the cascade subsystem, all run in-process
without standing up the FastAPI app:
- ``cascade sync [PATH]`` — flush the work queue. With ``PATH`` the
command first force-enqueues that single file (used after a manual
md edit when waiting for the watcher is impractical), then drains.
- ``cascade status`` — print the queue + LSN summary that the daemon
sees right now.
- ``cascade fix`` — list every ``failed`` row. With ``--apply``, also
reset ``retryable=TRUE`` rows back to ``pending`` and drain the
worker once so the retry actually runs before the command returns.
CLI is in-process (12 doc §7.1 + 16 doc §9.2): it constructs the same
:class:`CascadeOrchestrator` as the daemon but only calls
``sync_once`` / ``drain_once`` / ``queue_summary``. No watcher /
scanner background task is started.
"""
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Annotated
import typer
from sqlmodel import SQLModel
from everos.component.embedding import build_embedding_provider
from everos.component.tokenizer import build_tokenizer
from everos.component.utils.datetime import to_display_tz
from everos.config import load_settings
from everos.core.persistence import MemoryRoot
from everos.infra.persistence.lancedb import (
dispose_connection,
ensure_business_indexes,
get_connection,
verify_business_schemas,
)
from everos.infra.persistence.sqlite import (
dispose_engine,
get_engine,
md_change_state_repo,
)
from everos.memory.cascade import CascadeOrchestrator, match_kind
app = typer.Typer(
name="cascade",
help="Inspect and operate the md → LanceDB sync queue",
no_args_is_help=True,
)
# ── shared runtime context ───────────────────────────────────────────────
@asynccontextmanager
async def _runtime(): # type: ignore[no-untyped-def]
"""Stand up sqlite + lancedb the same way the API lifespan would.
The CLI piggybacks on the same singletons as the running daemon
(lazy + process-wide), so if a server happens to be running on
the same memory root, both share state correctly.
"""
engine = get_engine()
async with engine.begin() as conn:
await conn.run_sync(SQLModel.metadata.create_all)
await get_connection()
await verify_business_schemas()
await ensure_business_indexes()
try:
yield
finally:
await dispose_connection()
await dispose_engine()
def _build_orchestrator() -> CascadeOrchestrator:
settings = load_settings()
memory_root = MemoryRoot.default()
memory_root.ensure()
embedder = build_embedding_provider(settings.embedding)
tokenizer = build_tokenizer()
return CascadeOrchestrator(
memory_root=memory_root,
embedder=embedder,
tokenizer=tokenizer,
)
# ── sync ─────────────────────────────────────────────────────────────────
@app.command("sync")
def sync(
path: Annotated[
Path | None,
typer.Argument(
help="Optional md path to force-enqueue before draining. "
"If omitted, only the existing queue is drained.",
),
] = None,
) -> None:
"""Drain the cascade queue (and optionally re-enqueue a path first)."""
async def _run() -> None:
async with _runtime():
orchestrator = _build_orchestrator()
if path is not None:
rel = _resolve_relative(path)
spec = match_kind(rel)
if spec is None:
typer.echo(
f"error: path does not match any registered cascade "
f"kind: {rel}",
err=True,
)
raise typer.Exit(code=1)
await md_change_state_repo.force_enqueue(rel, spec.name)
typer.echo(f"force-enqueued {rel} (kind={spec.name})")
processed = await orchestrator.sync_once()
typer.echo(f"sync complete — processed {processed} row(s)")
asyncio.run(_run())
# ── status ───────────────────────────────────────────────────────────────
@app.command("status")
def status() -> None:
"""Print the queue / LSN summary."""
async def _run() -> None:
async with _runtime():
summary = await md_change_state_repo.queue_summary()
lag = max(0, summary.max_lsn - summary.last_processed_lsn)
typer.echo("queue:")
typer.echo(f" pending: {summary.pending}")
typer.echo(f" done: {summary.done}")
typer.echo(
f" failed (retryable=TRUE): {summary.failed_retryable}"
+ (
" (eligible for `cascade fix --apply`)"
if summary.failed_retryable
else ""
)
)
typer.echo(
f" failed (retryable=FALSE): {summary.failed_permanent}"
+ (
" (fix md and re-save to recover)"
if summary.failed_permanent
else ""
)
)
typer.echo("lsn:")
typer.echo(f" max: {summary.max_lsn}")
typer.echo(f" last_processed: {summary.last_processed_lsn}")
typer.echo(f" lag: {lag}")
asyncio.run(_run())
# ── fix ──────────────────────────────────────────────────────────────────
@app.command("fix")
def fix(
apply: Annotated[
bool,
typer.Option(
"--apply",
help="Re-enqueue every `retryable=TRUE` row and drain the worker.",
),
] = False,
) -> None:
"""List failed rows (default) or re-enqueue retryable ones (``--apply``)."""
async def _run() -> None:
async with _runtime():
rows = await md_change_state_repo.list_failed()
if not rows:
typer.echo("no failed rows")
return
if not apply:
_print_failed_table(rows)
retryable = sum(1 for r in rows if r.retryable)
permanent = sum(1 for r in rows if not r.retryable)
typer.echo("")
if retryable:
typer.echo(
f"run `everos cascade fix --apply` to re-enqueue "
f"the {retryable} retryable row(s)."
)
if permanent:
typer.echo(
f"the {permanent} retryable=FALSE row(s) require "
"editing the md and re-saving."
)
return
moved = await md_change_state_repo.reset_retryable_to_pending()
typer.echo(f"re-enqueued {moved} retryable row(s)")
if moved:
orchestrator = _build_orchestrator()
processed = await orchestrator.drain_once()
typer.echo(f"[worker] processed {processed} row(s) on drain")
permanent_rows = [r for r in rows if not r.retryable]
if permanent_rows:
typer.echo(
f"{len(permanent_rows)} retryable=FALSE row(s) left untouched:"
)
for r in permanent_rows:
typer.echo(f" {r.md_path}")
asyncio.run(_run())
# ── helpers ──────────────────────────────────────────────────────────────
def _resolve_relative(p: Path) -> str:
"""Translate an absolute / relative path arg into the memory-root rel form.
The state table stores paths relative to memory root, so the CLI
must match that convention before calling :meth:`force_enqueue`.
Outside-the-root inputs surface as an error in the caller.
"""
memory_root = MemoryRoot.default()
absolute = p.expanduser().resolve()
try:
rel = absolute.relative_to(memory_root.root)
except ValueError as exc:
raise typer.BadParameter(
f"path {p!s} is not under memory root {memory_root.root!s}"
) from exc
return rel.as_posix()
def _print_failed_table(rows: list) -> None: # type: ignore[type-arg]
headers = ("md_path", "retryable", "retries", "last_attempt", "error")
widths = [
max(len(headers[0]), max(len(r.md_path) for r in rows)),
len(headers[1]),
len(headers[2]),
len(headers[3]),
max(len(headers[4]), max(len(r.error or "") for r in rows)),
]
fmt = " ".join(f"{{:<{w}}}" for w in widths)
typer.echo(f"{len(rows)} failed row(s):\n")
typer.echo(fmt.format(*headers))
for r in rows:
typer.echo(
fmt.format(
r.md_path,
"TRUE" if r.retryable else "FALSE",
r.retry_count,
to_display_tz(r.last_attempt_at).isoformat()
if r.last_attempt_at
else "",
r.error or "",
)
)

183
src/everos/entrypoints/cli/commands/init_cmd.py Normal file
View File

@ -0,0 +1,183 @@
"""``everos init`` — generate a starter ``.env`` from the packaged template.
The ``env.template`` ships inside the wheel as package data at
``everos/templates/env.template``. ``init`` reads it via
:mod:`importlib.resources`, so the command works identically for pip-
installed users and source-tree users (the file is the single source
of truth).
Subcommand mounted as ``everos init`` (top-level leaf command — not a
Typer group), to match the idiomatic ``alembic init`` / ``django-admin
startproject`` shape.
"""
from __future__ import annotations
import contextlib
import logging
import os
import sys
import tempfile
from importlib import resources
from pathlib import Path
import typer
_TEMPLATE_PACKAGE = "everos.templates"
_TEMPLATE_NAME = "env.template"
_log = logging.getLogger("everos.cli.init")
def _read_template() -> str:
"""Read the packaged ``env.template`` from wheel resources.
Returns the file contents as a UTF-8 string. Raises ``RuntimeError``
on missing-file — if this fires it means the wheel was built from a
source tree where ``src/everos/templates/env.template`` was missing
(canonical location; auto-included via ``packages=["src/everos"]``
in ``pyproject.toml``).
"""
try:
return (
resources.files(_TEMPLATE_PACKAGE)
.joinpath(_TEMPLATE_NAME)
.read_text(encoding="utf-8")
)
except (FileNotFoundError, ModuleNotFoundError) as exc:
raise RuntimeError(
f"packaged template {_TEMPLATE_NAME!r} not found under "
f"{_TEMPLATE_PACKAGE!r}; the wheel is missing its "
"force-include entry (see pyproject.toml "
"[tool.hatch.build.targets.wheel.force-include])."
) from exc
def _xdg_default_path() -> Path:
"""``$XDG_CONFIG_HOME/everos/.env`` (default ``~/.config/everos/.env``)."""
xdg = os.environ.get("XDG_CONFIG_HOME") or "~/.config"
return Path(xdg).expanduser() / "everos" / ".env"
def _atomic_write(target: Path, content: str, mode: int = 0o600) -> None:
"""Write ``content`` to ``target`` atomically with ``mode`` permission.
Writes to a tempfile in the same directory then ``os.replace``s it
onto the target — guarantees either the full new file is visible or
the original (if any) is untouched. Permission bits applied before
the rename so the file is never readable by other users.
"""
target.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(
prefix=target.name + ".",
dir=target.parent,
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write(content)
os.chmod(tmp_path, mode)
os.replace(tmp_path, target)
except Exception:
with contextlib.suppress(OSError):
os.unlink(tmp_path)
raise
def register(parent: typer.Typer) -> None:
"""Attach the ``init`` command to the root CLI app."""
@parent.command("init")
def init(
to: str | None = typer.Option(
None,
"--to",
help=(
"Target path for the .env file (default: ./.env). "
"Parent directories are created if needed."
),
),
force: bool = typer.Option(
False,
"--force",
help="Overwrite an existing file at the target path.",
),
print_: bool = typer.Option(
False,
"--print",
help="Print the template to stdout instead of writing to disk.",
),
xdg: bool = typer.Option(
False,
"--xdg",
help=(
"Shortcut for --to=${XDG_CONFIG_HOME:-~/.config}/everos/.env "
"(mutually exclusive with --to)."
),
),
) -> None:
"""Generate a starter ``.env`` from the packaged template.
Common flows::
everos init # writes ./.env
everos init --xdg # writes ~/.config/everos/.env
everos init --to /etc/foo.env --force
everos init --print > custom.env
Exit codes:
- 0 — written successfully (or printed to stdout).
- 1 — target file already exists and ``--force`` was not given.
- 2 — packaged template missing (wheel build problem).
- 3 — write failed (permissions / disk full / parent unwritable).
"""
if xdg and to is not None:
typer.secho(
"error: --xdg and --to are mutually exclusive",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(code=2)
try:
template = _read_template()
except RuntimeError as exc:
typer.secho(f"error: {exc}", fg=typer.colors.RED, err=True)
raise typer.Exit(code=2) from exc
if print_:
sys.stdout.write(template)
return
if xdg:
target = _xdg_default_path()
elif to is not None:
target = Path(to).expanduser().resolve()
else:
target = Path.cwd() / ".env"
if target.exists() and not force:
typer.secho(
f"error: {target} already exists; pass --force to overwrite",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(code=1)
try:
_atomic_write(target, template)
except OSError as exc:
typer.secho(
f"error: failed to write {target}: {exc}",
fg=typer.colors.RED,
err=True,
)
raise typer.Exit(code=3) from exc
# Friendly next-step block (stdout — quiet enough for piping).
size_kb = target.stat().st_size / 1024
typer.secho(f"✓ wrote {target} ({size_kb:.1f} KB)", fg=typer.colors.GREEN)
typer.echo("Next steps:")
typer.echo(" 1. Edit the file and fill in the API keys (see comments inside).")
typer.echo(" 2. Run `everos server start`.")
typer.echo("Docs: https://github.com/evermind/everos/blob/master/QUICKSTART.md")

161
src/everos/entrypoints/cli/commands/server.py Normal file
View File

@ -0,0 +1,161 @@
"""``everos server`` subcommand group.
Provides ``everos server start`` to run the HTTP API via uvicorn. CLI
parses arguments, configures structured logging, then hands off to
uvicorn pointing at :func:`everos.entrypoints.api.app.create_app` as a
factory.
"""
from __future__ import annotations
import logging
import os
import sys
from pathlib import Path
import typer
import uvicorn
app = typer.Typer(
name="server",
help="Run / manage the HTTP API server",
no_args_is_help=True,
)
def _resolve_env_file(explicit: str | None) -> Path | None:
"""Find the first existing ``.env`` along the four-layer search path.
Search order (highest-wins):
1. ``explicit`` — when the caller passed ``--env-file <path>``.
2. ``./.env`` — the current working directory (project-local convention).
3. ``${XDG_CONFIG_HOME:-~/.config}/everos/.env`` — XDG-standard user config.
4. ``~/.everos/.env`` — the project's default memory-root location.
Returns ``None`` if none of the layers exist (caller may then fall back
to inherited process env / CI secrets).
"""
candidates: list[Path] = []
if explicit:
candidates.append(Path(explicit).expanduser())
candidates.append(Path.cwd() / ".env")
xdg = os.environ.get("XDG_CONFIG_HOME") or "~/.config"
candidates.append(Path(xdg).expanduser() / "everos" / ".env")
candidates.append(Path("~/.everos/.env").expanduser())
for p in candidates:
try:
if p.is_file():
return p
except OSError:
# Path traversal / permission denied on a fallback candidate
# must not crash the search — skip and keep going.
continue
return None
def _load_env_file(path: str | None) -> Path | None:
"""Load environment variables from the resolved ``.env`` file.
Returns the path that was loaded, or ``None`` when no ``.env`` was
found anywhere along the search path. Existence of a ``.env`` is
optional — the user may rely entirely on inherited process env
(e.g. container / CI secret injection).
"""
resolved = _resolve_env_file(path)
if resolved is None:
return None
try:
from dotenv import load_dotenv
load_dotenv(resolved, override=False)
except ImportError:
# python-dotenv is in our deps; tolerate its absence anyway.
pass
return resolved
@app.command("start")
def start(
host: str | None = typer.Option(
None,
"--host",
help="Bind host (env: EVEROS_API__HOST, default: 127.0.0.1)",
),
port: int | None = typer.Option(
None,
"--port",
help="Bind port (env: EVEROS_API__PORT, default: 8000)",
),
env_file: str | None = typer.Option(
None,
"--env-file",
help=(
"Path to a dotenv file (highest priority). When omitted, "
"the server searches: ./.env → ${XDG_CONFIG_HOME:-~/.config}"
"/everos/.env → ~/.everos/.env. Run `everos init` to create one."
),
),
reload: bool = typer.Option(
False,
"--reload",
help="Reload on source changes (development)",
),
log_level: str | None = typer.Option(
None,
"--log-level",
help="Log level (env: EVEROS_LOG_LEVEL, default: INFO)",
),
) -> None:
"""Start the HTTP API server."""
loaded_env = _load_env_file(env_file)
# Load settings AFTER .env is in place so EVEROS_API__HOST and
# EVEROS_API__PORT (and any other env override) are honored.
from everos.config import load_settings
settings = load_settings()
host_resolved = host or settings.api.host
port_resolved = port if port is not None else settings.api.port
log_level_resolved = (log_level or os.getenv("EVEROS_LOG_LEVEL", "INFO")).upper()
from everos.core.observability.logging import configure_logging
configure_logging(level=log_level_resolved)
bootstrap_logger = logging.getLogger("everos.cli.server")
if loaded_env is not None:
bootstrap_logger.info("loaded env file: %s", loaded_env)
else:
bootstrap_logger.info(
"no .env found along the search path; relying on inherited env vars "
"(run `everos init` to generate one)"
)
bootstrap_logger.info("starting everos on %s:%d", host_resolved, port_resolved)
if host_resolved == "0.0.0.0":
bootstrap_logger.warning(
"binding to 0.0.0.0 exposes the API on all interfaces; EverOS "
"ships no built-in auth — see SECURITY.md"
)
try:
uvicorn.run(
"everos.entrypoints.api.app:create_app",
host=host_resolved,
port=port_resolved,
reload=reload,
factory=True,
log_level=log_level_resolved.lower(),
# ``configure_logging()`` above already installed the root
# handler + structlog ProcessorFormatter. ``log_config=None``
# stops uvicorn from running its own ``dictConfig`` over
# ours; otherwise uvicorn / fastapi messages revert to the
# ``INFO:`` no-structlog format on every restart.
log_config=None,
)
except KeyboardInterrupt:
bootstrap_logger.info("interrupted; shutting down")
except (OSError, RuntimeError) as exc:
bootstrap_logger.error("startup failed: %s", exc)
sys.exit(1)