From 9758ba264fe173958205b5f9674fc6a79cef6628 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 9 Jul 2026 03:30:55 +0000 Subject: [PATCH 1/3] feat(index): add optional Milvus derived index backend Signed-off-by: Cheney Zhang --- QUICKSTART.md | 8 +- README.md | 16 +- config.example.toml | 16 + docs/api.md | 15 +- docs/architecture.md | 26 +- docs/cascade_runbook.md | 16 +- docs/cli.md | 15 +- docs/configuration.md | 22 + docs/how-memory-works.md | 49 +- docs/knowledge.md | 18 +- docs/multimodal.md | 2 +- docs/overview.md | 6 +- docs/reflection.md | 11 +- docs/storage_layout.md | 43 +- pyproject.toml | 13 +- src/everos/README.md | 2 +- src/everos/config/__init__.py | 7 +- src/everos/config/default.toml | 19 +- src/everos/config/settings.py | 43 + src/everos/core/persistence/memory_root.py | 13 + .../entrypoints/api/lifespans/lancedb.py | 40 +- .../entrypoints/cli/commands/cascade.py | 18 +- .../infra/persistence/index/__init__.py | 329 ++++++++ .../infra/persistence/milvus/__init__.py | 49 ++ .../persistence/milvus/milvus_manager.py | 87 ++ src/everos/infra/persistence/milvus/repos.py | 119 +++ .../infra/persistence/milvus/repository.py | 750 ++++++++++++++++++ .../memory/cascade/handlers/agent_case.py | 2 +- .../memory/cascade/handlers/agent_skill.py | 2 +- .../memory/cascade/handlers/atomic_fact.py | 2 +- src/everos/memory/cascade/handlers/episode.py | 2 +- .../memory/cascade/handlers/foresight.py | 2 +- .../cascade/handlers/knowledge_topic.py | 2 +- .../memory/cascade/handlers/user_profile.py | 2 +- src/everos/memory/cascade/registry.py | 2 +- src/everos/memory/get/filters_adapter.py | 19 + src/everos/memory/get/manager.py | 12 +- src/everos/memory/search/filters.py | 215 ++++- src/everos/memory/search/manager.py | 7 +- src/everos/memory/search/recall/agent_case.py | 56 +- .../memory/search/recall/agent_skill.py | 57 +- .../memory/search/recall/atomic_fact.py | 39 +- src/everos/memory/search/recall/episode.py | 36 +- .../memory/search/recall/knowledge_topic.py | 39 +- src/everos/memory/search/recall/profile.py | 2 +- .../memory/strategies/extract_agent_skill.py | 8 +- .../memory/strategies/extract_user_profile.py | 2 +- .../memory/strategies/reflect_episodes.py | 2 +- src/everos/service/get.py | 6 +- tests/unit/test_config/test_settings.py | 27 + .../test_persistence/test_memory_root.py | 3 + .../unit/test_infra/test_milvus/test_repo.py | 127 +++ .../unit/test_memory/test_get/test_manager.py | 4 +- .../test_memory/test_search/test_filters.py | 33 + .../test_memory/test_search/test_manager.py | 16 +- .../test_search/test_recall_episode.py | 68 +- .../test_recall_knowledge_topic.py | 121 ++- uv.lock | 248 +++++- 58 files changed, 2436 insertions(+), 479 deletions(-) create mode 100644 src/everos/infra/persistence/index/__init__.py create mode 100644 src/everos/infra/persistence/milvus/__init__.py create mode 100644 src/everos/infra/persistence/milvus/milvus_manager.py create mode 100644 src/everos/infra/persistence/milvus/repos.py create mode 100644 src/everos/infra/persistence/milvus/repository.py create mode 100644 tests/unit/test_infra/test_milvus/test_repo.py diff --git a/QUICKSTART.md b/QUICKSTART.md index 5d5f8ab78..f2ab57f63 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -81,8 +81,9 @@ works. ## 3. Start the server -Check your file descriptor limit — EverOS opens many LanceDB segment -files under concurrent search + indexing. Platform defaults: +With the default LanceDB index backend, check your file descriptor limit — +EverOS opens many LanceDB segment files under concurrent search + indexing. +Platform defaults: **macOS 256** · **Linux 1024** · **Windows 8192**. If yours is below 4096, raise it before starting: @@ -258,7 +259,8 @@ This is what makes EverOS different — memory persists as plain Markdown: ├── ome.toml ← strategy config (hot-reloaded) ├── .index/ ← derived indexes (rebuildable from md) │ ├── sqlite/system.db -│ └── lancedb/ +│ ├── lancedb/ ← default derived index backend +│ └── milvus/ ← Milvus Lite DB when configured └── .tmp/ ``` diff --git a/README.md b/README.md index f423d467a..905fc8e9d 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,10 @@ EverOS is a Python library and local-first memory runtime for agents and makers. It gives one portable memory layer across coding assistants, apps, devices, and workflows from day one. It stores conversations, files, and agent -trajectories as readable Markdown, then syncs local SQLite and LanceDB indexes -for fast retrieval and self-evolving reuse. +trajectories as readable Markdown, then syncs local SQLite and a derived +vector/BM25 index for fast retrieval and self-evolving reuse. LanceDB is the +default embedded index backend; Milvus can be enabled when you want the same +rebuildable index to run on Milvus Lite, Milvus server, or Zilliz Cloud. @@ -60,7 +62,7 @@ for fast retrieval and self-evolving reuse. - + @@ -114,6 +116,12 @@ uv pip install everos # or: pip install everos ``` +To use Milvus as the derived index backend: + +```bash +uv pip install "everos[milvus]" +``` + ### 2. Play With The Demo Run this before configuring API keys or starting the server: @@ -642,7 +650,7 @@ Explore stored entities and relationships in a graph interface. Frontend demo; b ## Documentation - [docs/everos-demo.md](docs/everos-demo.md) — Demo scope and TUI source layout -- [docs/how-memory-works.md](docs/how-memory-works.md) — Markdown, SQLite, LanceDB, and recall flow +- [docs/how-memory-works.md](docs/how-memory-works.md) — Markdown, SQLite, derived index backends, and recall flow - [docs/use-cases.md](docs/use-cases.md) — Full use-case gallery and integration examples - [docs/engineering.md](docs/engineering.md) — Contributor engineering reference: build, test, CI, conventions - [docs/migration-to-1.0.0.md](docs/migration-to-1.0.0.md) — Legacy API migration notes diff --git a/config.example.toml b/config.example.toml index f8043127f..0145dd686 100644 --- a/config.example.toml +++ b/config.example.toml @@ -56,3 +56,19 @@ max_concurrent = 5 # # [lancedb] # read_consistency_seconds = 5.0 + +# ── Optional Milvus derived index ───────────────────── +# Markdown remains the source of truth. Set the index backend to Milvus +# when you want the rebuildable vector/BM25 index to live in Milvus. +# Leave uri empty for Milvus Lite at /.index/milvus/milvus.db. +# +# [index] +# backend = "milvus" +# +# [milvus] +# uri = "" # or "http://localhost:19530" +# token = "" # required for Zilliz Cloud/auth-enabled Milvus +# db_name = "" +# consistency_level = "Session" +# dimension = 1024 # must match your embedding model output +# collection_prefix = "everos" diff --git a/docs/api.md b/docs/api.md index b76ca99fb..19e68d803 100644 --- a/docs/api.md +++ b/docs/api.md @@ -84,8 +84,9 @@ bare FastAPI `detail`); see [Errors](#errors). `/add` and `/flush` write the markdown file (the source of truth) **synchronously** — when the call returns with `status: "extracted"`, -the new entry exists on disk. The LanceDB vector / BM25 / scalar index -is rebuilt by the in-process **cascade coroutine asynchronously**. +the new entry exists on disk. The configured vector / BM25 / scalar +index backend is rebuilt by the in-process **cascade coroutine +asynchronously**. That means `/search` and `/get` may not see a record immediately after the `/flush` that produced it. Typical sync latency is sub-second, but @@ -362,7 +363,7 @@ A recursive boolean tree of predicates. Used by `/search.filters` and `/get.filters`. The Pydantic envelope only checks the recursive combinator shape; field-level validity (which scalar fields are filterable, which operators apply, value coercion) runs when the -node is compiled to a LanceDB `where` clause server-side. Compile +node is compiled to a backend-specific filter clause server-side. Compile errors surface as `422` with the offending field / operator in `error.message`. @@ -455,12 +456,12 @@ Examples: |---|---| | `"keyword"` | BM25 only — pure lexical match, no embedding cost | | `"vector"` | Dense vector ANN only — semantic recall, no lexical | -| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter in a single LanceDB query | +| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter against the configured derived index backend | | `"agentic"` | Iterative cluster-path retrieval driven by a cross-encoder rerank loop; higher quality at higher latency / cost | `"hybrid"` is the default because it balances recall and precision -with one LanceDB roundtrip. `"agentic"` calls the LLM in a loop and -should be reserved for offline or background workflows. +without requiring the agentic loop. `"agentic"` calls the LLM in a loop +and should be reserved for offline or background workflows. ### GetMemoryType @@ -596,7 +597,7 @@ scope. this `(session_id, app_id, project_id)`, or it was already flushed). `/flush` is synchronous with respect to markdown persistence: by the -time the response returns, the new entry is on disk. LanceDB index +time the response returns, the new entry is on disk. Derived index sync is still asynchronous — see [Eventual consistency](#eventual-consistency). diff --git a/docs/architecture.md b/docs/architecture.md index afb5efe30..4aae7826a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ │ + reflection + strategies + get + events │ ├──────────────────────────────────────────────────────┤ │ infra/persistence (Storage adapters; infra/ may host other adapter types) │ -│ markdown + sqlite + lancedb │ +│ markdown + sqlite + derived index │ └──────────────────────────────────────────────────────┘ Cross-cutting (used by all layers, depends on none): @@ -68,8 +68,8 @@ layers = [ └────────────────────────────────────────────────────────────────┘ ┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ - │ Markdown │ │ SQLite │ │ LanceDB │ - │ (truth) │ │ (state) │ │ (index) │ + │ Markdown │ │ SQLite │ │ Derived index │ + │ (truth) │ │ (state) │ │ LanceDB/Milvus │ ├──────────────┤ ├──────────────┤ ├─────────────────┤ │ entries + │ │ change queue │ │ vector ANN │ │ frontmatter │ │ + state/LSN │ │ BM25 (Tantivy) │ @@ -78,7 +78,7 @@ layers = [ └──────────────┘ └──────────────┘ └─────────────────┘ │ │ │ ▼ ▼ ▼ - memory-root/ .index/sqlite/ .index/lancedb/ + memory-root/ .index/sqlite/ .index// (truth source) (system data) (rebuildable) ``` @@ -101,10 +101,13 @@ External message │ │ ▼ ▼ 4a. SQLite 4b. memory.cascade (async daemon) - audit watches md → diff entries → LanceDB sync + audit watches md → diff entries → index sync ``` -**Key guarantee**: md write is strongly consistent (fsync). LanceDB is eventually consistent. LanceDB unavailability does not block response — changes buffer in the SQLite `md_change_state` queue, replayed on recovery. +**Key guarantee**: md write is strongly consistent (fsync). The derived +index is eventually consistent. Index backend unavailability does not block +response — changes buffer in the SQLite `md_change_state` queue, replayed on +recovery. ## Read path @@ -115,8 +118,8 @@ User query 1. service.search │ ▼ -2. memory.search (hybrid) single LanceDB query = - BM25 + vector ANN + scalar filter +2. memory.search (hybrid) BM25 + vector ANN + scalar filter + through the configured index backend │ ▼ 3. (optional) read md original markdown for context @@ -139,12 +142,13 @@ extract/ ### `memory/cascade/` -Daemon that watches markdown changes and syncs to LanceDB: +Daemon that watches markdown changes and syncs to the configured derived +index backend: - inotify / FSEvents file watcher (cross-platform via `watchdog`) - 500ms debounce - Entry-level diff (added / changed / removed) -- LanceDB single-transaction update (text + vector columns atomic) +- Per-entry index upsert / delete (text + vector columns update together) - LSN-based crash recovery via the SQLite `md_change_state` queue - Handlers for all eight business kinds: episode, atomic_fact, foresight, user_profile, agent_case, agent_skill, knowledge_document, knowledge_topic @@ -225,7 +229,7 @@ holding **only memory extraction algorithms**: everalgo is: - **Stateless** — pure functions, no class hierarchy -- **No I/O** — does not touch md files / LanceDB / SQLite +- **No I/O** — does not touch md files, derived indexes, or SQLite - **No prompts inline** — extractors that accept a prompt-override parameter use the project-supplied value; others use their algo-bundled defaults This boundary lets everalgo be reused across product forms (this open-source build, EverOS Cloud, OpenClaw plugins, etc.). diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index c9811eeea..d7bb9d530 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -1,9 +1,13 @@ # Cascade Runbook -The cascade daemon keeps LanceDB in sync with the markdown files under -the memory root. Service / entry points only ever write markdown; the -daemon is the **sole** writer of the LanceDB index. This runbook covers -the recurring operational questions. +The cascade daemon keeps the configured derived index in sync with the +markdown files under the memory root. Service / entry points only ever write +markdown; the daemon is the **sole** writer of the derived index. This runbook +covers the recurring operational questions. + +Sections that mention LanceDB-specific schemas, index cache, file descriptors, +or `lance error` messages apply to the default LanceDB backend. Milvus uses the +same cascade queue with backend-specific collection management. ## What runs where @@ -13,7 +17,7 @@ providers in order: 1. **Metrics** — Prometheus collector. 2. **LLM** — LLM client initialisation. 3. **SQLite** — system DB + schema (`SQLModel.metadata.create_all`). -4. **LanceDB** — async connection + schema verification + FTS indexes. +4. **Derived index** — async connection + schema verification + search indexes. 5. **Cascade** — watcher + scanner + worker, all in-process tasks. 6. **OME** — offline memory engine. @@ -23,7 +27,7 @@ The cascade subsystem itself is three independent loops: |---|---|---| | Watcher | `watchdog` filesystem events (sync thread) | `md_change_state.upsert` per registered kind | | Scanner | Periodic walk (`scan_interval_seconds`, default 30 s) | Same — catches changes the watcher missed | -| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → LanceDB upsert / delete | +| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → index upsert / delete | Every loop talks to the same `md_change_state` sqlite table. The worker's claim mode (`pending → processing → done/failed`) keeps diff --git a/docs/cli.md b/docs/cli.md index e24e6e5ea..c56574ada 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,7 +3,7 @@ The `everos` command-line entry point covers **setup and operations** — generate starter config files (`init`), run the HTTP API server (`server start`), inspect effective config (`config show`), and operate the -md → LanceDB index queue (`cascade`). Hot-path +md → derived index queue (`cascade`). Hot-path business (`/add` `/flush` `/search` `/get`) is the **HTTP API**, not the CLI. @@ -32,7 +32,7 @@ everos │ └── show [--root PATH] Show effective configuration ├── server │ └── start [--host] [--port] [--root] [--reload] [--log-level] Start the HTTP API server (uvicorn) -└── cascade [--root PATH] Inspect / operate the md → LanceDB sync queue +└── cascade [--root PATH] Inspect / operate the md → derived index sync queue ├── status Queue / LSN summary ├── sync [PATH] Drain the queue now (optional PATH force-enqueues) └── fix [--apply] List failed rows / re-enqueue retryable ones @@ -43,8 +43,9 @@ Each subcommand lives in its own module under registered in `cli/main.py`. The CLI is intentionally small — hot-path business (`/add` `/flush` `/search` `/get`) is the **HTTP API**, not the CLI; the CLI covers setup (`init`), running the server, and index ops -(`cascade`). There is no `reindex` command — rebuild by deleting -`/.index/lancedb` and restarting, or run `everos cascade sync`. +(`cascade`). There is no `reindex` command — rebuild by deleting the +selected backend's derived index directory (`/.index/lancedb` or +`/.index/milvus`) and restarting, or run `everos cascade sync`. ## `everos server start` @@ -68,8 +69,8 @@ everos server start \ | `--root` | `EVEROS_ROOT` | `~/.everos` | | `--reload` | — | off (use in development) | -Lifespan startup wires the storage backends (SQLite engine + LanceDB -connection) on app boot; see +Lifespan startup wires the storage backends (SQLite engine + configured +derived index backend) on app boot; see [`entrypoints/api/lifespans/`](../src/everos/entrypoints/api/lifespans/). ## Configuration via env vars @@ -81,7 +82,9 @@ Both CLI and HTTP server read configuration from `pydantic-settings`: | `EVEROS_ROOT` | memory-root path (default `~/.everos`) | | `EVEROS_MEMORY__TIMEZONE` | `Settings.memory.timezone` (e.g. `Asia/Shanghai`) | | `EVEROS_SQLITE__BUSY_TIMEOUT_MS` | `Settings.sqlite.busy_timeout_ms` | +| `EVEROS_INDEX__BACKEND` | `Settings.index.backend` (`lancedb` or `milvus`) | | `EVEROS_LANCEDB__READ_CONSISTENCY_SECONDS` | `Settings.lancedb.read_consistency_seconds` | +| `EVEROS_MILVUS__URI` | `Settings.milvus.uri` | Pattern: `EVEROS_
__` (double underscore = nesting). See [`config/settings.py`](../src/everos/config/settings.py). diff --git a/docs/configuration.md b/docs/configuration.md index 98fdf3479..6848d7839 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,26 @@ everos init --root /data/everos | `read_consistency_seconds` | float \| null | `null` | Read consistency interval. `null` = no check, `0` = strict, `>0` = eventual. | | `index_cache_size_bytes` | int | `16777216` | Upper bound on LanceDB index cache (16 MB default). | +### `[index]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `backend` | `"lancedb"` \| `"milvus"` | `"lancedb"` | Rebuildable vector/BM25 index backend used by cascade, search, and get. Markdown remains the source of truth; SQLite remains the system state store. | + +### `[milvus]` + +Milvus is optional. Install with `everos[milvus]`, then set +`[index].backend = "milvus"`. + +| Field | Type | Default | Description | +|---|---|---|---| +| `uri` | string \| null | `""` | Empty uses Milvus Lite at `/.index/milvus/milvus.db`. Set to a Milvus server or Zilliz Cloud endpoint to use an external service. | +| `token` | string \| null | `""` | Token for Zilliz Cloud or auth-enabled Milvus. | +| `db_name` | string \| null | `""` | Optional Milvus database name. | +| `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | +| `dimension` | int | `1024` | Dense vector dimension. Must match the configured embedding model output. | +| `collection_prefix` | string | `"everos"` | Prefix for EverOS Milvus collections. | + ### `[llm]` | Field | Type | Default | Required | Description | @@ -228,3 +248,5 @@ Examples: | `[llm] api_key = "sk-..."` | `EVEROS_LLM__API_KEY=sk-...` | | `[sqlite] busy_timeout_ms = 10000` | `EVEROS_SQLITE__BUSY_TIMEOUT_MS=10000` | | `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | +| `[index] backend = "milvus"` | `EVEROS_INDEX__BACKEND=milvus` | +| `[milvus] uri = "http://localhost:19530"` | `EVEROS_MILVUS__URI=http://localhost:19530` | diff --git a/docs/how-memory-works.md b/docs/how-memory-works.md index 1a8136478..b3d25ba1d 100644 --- a/docs/how-memory-works.md +++ b/docs/how-memory-works.md @@ -23,14 +23,16 @@ This is the narrative companion to the reference docs: see ## The storage stack -Three embedded pieces, each owning what it is best at. Markdown is the -**source of truth**; the other two are **derived and rebuildable**. +Three pieces, each owning what it is best at. Markdown is the +**source of truth**; the other layers are **derived and rebuildable**. +By default the vector/BM25 index is embedded LanceDB. When configured, +Milvus can serve the same derived index role. | Layer | Backed by | Holds | Rebuildable? | |---|---|---|---| | **Markdown + YAML frontmatter** | plain `.md` files | the memory content itself — the only portable, human-editable asset | — (it *is* the truth) | | **SQLite** (`aiosqlite`) | `.index/sqlite/*.db` | system state, audit log, the cascade queue, the boundary buffer, OME engine state | ✅ from markdown | -| **LanceDB** (Arrow) | `.index/lancedb/*.lance` | vector + BM25 + scalar columns for retrieval | ✅ from markdown | +| **Derived index** | `.index/lancedb/*.lance` by default, or `.index/milvus/milvus.db` for Milvus Lite | vector + BM25 + scalar columns for retrieval | ✅ from markdown | !!! note "The one rule that follows from this" Delete the entire `.index/` directory and **no memory is lost** — it @@ -77,8 +79,10 @@ visually distinct from a user-named one). │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) -│ └── lancedb/ -│ └── .lance/ one Arrow table per kind +│ ├── lancedb/ +│ │ └── .lance/ default derived index backend +│ └── milvus/ +│ └── milvus.db Milvus Lite when configured │ ├── ome.toml ← user-editable OME strategy overrides (hot-reloaded) └── .tmp/ atomic-write staging @@ -96,7 +100,7 @@ visually distinct from a user-named one). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py); every path above is a property on it. `MemoryRoot.ensure()` creates the runtime dirs -(`.index/{sqlite,lancedb}/`, `.tmp/`); user-visible dirs appear on first +(`.index/{sqlite,lancedb,milvus}/`, `.tmp/`); user-visible dirs appear on first write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. ## How a memory is born @@ -129,7 +133,7 @@ index catches up asynchronously. ▼ md_change_state queue (SQLite, durable) ▼ - rebuild LanceDB rows ──▶ searchable + rebuild index rows ──▶ searchable ``` - **`/add`** appends messages to a per-`(session_id, app_id, project_id)` @@ -142,7 +146,7 @@ index catches up asynchronously. - Everything else (atomic facts, foresight, profile, agent cases/skills) is produced **asynchronously** by the OME — see [the OME section](#the-offline-memory-engine-ome). -- The **cascade daemon** turns every `.md` write into LanceDB rows so the +- The **cascade daemon** turns every `.md` write into derived index rows so the content becomes searchable. ## Memory types & storage strategies @@ -177,7 +181,8 @@ The three strategies: ## The cascade daemon -The cascade subsystem keeps LanceDB in sync with the markdown tree. It runs +The cascade subsystem keeps the configured derived index in sync with the +markdown tree. It runs **in-process** with the server (a coroutine started by the app lifespan), not as a separate OS daemon. @@ -187,7 +192,7 @@ not as a separate OS daemon. durable, so a crash mid-sync replays on restart. 3. A worker drains the queue at **entry-level** granularity: it diffs the file, re-embeds only changed entries (keyed by `content_sha256`), and - upserts the LanceDB rows. + upserts the configured index rows. Because markdown is the source of truth, **editing a file directly is fully supported** — open an episode in VSCode / Obsidian / Vim, change an @@ -257,8 +262,8 @@ Two paths, two guarantees: | Path | Guarantee | Detail | |---|---|---| -| **Write** (`/add`, `/flush`) | **strong** | the episode `.md` is on disk before the call returns `extracted`; never blocks on LanceDB | -| **Read** (`/search`, `/get`) | **eventual** | reads LanceDB, which lags md by the cascade processing time — sub-second typically, up to ~10–15 s under load | +| **Write** (`/add`, `/flush`) | **strong** | the episode `.md` is on disk before the call returns `extracted`; never blocks on the derived index | +| **Read** (`/search`, `/get`) | **eventual** | reads the configured derived index, which lags md by the cascade processing time — sub-second typically, up to ~10–15 s under load | So a `/search` immediately after the `/flush` that produced a record may miss it. The markdown is durable regardless; index lag never loses data. If @@ -272,17 +277,21 @@ entry needs re-embedding; an LSN watermark (in `system.db`) orders rebuilds; the durable `md_change_state` queue is the replayable audit trail. -## Zero external services +## Zero external services by default No database server, message broker, or vector service to run. Vector ANN, -full-text BM25, and scalar filtering all execute inside the **embedded -LanceDB** engine in one query; SQLite is a local file. The whole stack is a -single directory you can copy, back up, or check the user-visible parts of -into git. +full-text BM25, and scalar filtering execute inside the embedded LanceDB +backend by default; SQLite is a local file. The whole default stack is a single +directory you can copy, back up, or check the user-visible parts of into git. + +If you set `[index].backend = "milvus"`, EverOS uses Milvus as the same +rebuildable derived index. Empty `[milvus].uri` uses Milvus Lite under the +memory root; setting `uri` / `token` points the index at Milvus server or +Zilliz Cloud. !!! note There is no automatic "grep over markdown" search fallback today — if - the LanceDB index is unavailable, rebuild it from markdown (it is + the configured index is unavailable, rebuild it from markdown (it is derived and disposable) rather than relying on a degraded search path. ## Operating it @@ -294,12 +303,12 @@ The CLI ([cli.md](cli.md)) is intentionally small: | `everos init` | generate starter config files (`everos.toml` + `ome.toml`) | | `everos server start` | run the HTTP API (cascade + OME start with it) | | `everos cascade status` | queue / LSN summary | -| `everos cascade sync` | drain the cascade queue now (force md → LanceDB) | +| `everos cascade sync` | drain the cascade queue now (force md → derived index) | | `everos cascade fix` | list failed rows / re-enqueue retryable ones | !!! warning "There is no `everos reindex` or `everos flush`" - **Reindex** = the index is rebuildable: stop the server, - `rm -rf /.index/lancedb`, restart — the cascade + `rm -rf /.index/lancedb` (or `.index/milvus` for Milvus Lite), restart — the cascade rebuilds from markdown. For an incremental catch-up, use `everos cascade sync`. - **Flush** is an HTTP endpoint (`POST /api/v1/memory/flush`), not a diff --git a/docs/knowledge.md b/docs/knowledge.md index c2aa81698..11f40ca70 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -48,8 +48,8 @@ Each level corresponds to a different granularity of API: ## Storage layout Every document is a self-contained directory. Markdown files are the -single source of truth; SQLite and LanceDB are derived indexes built -automatically by the cascade daemon. +single source of truth; SQLite and the configured vector/BM25 index are +derived indexes built automatically by the cascade daemon. ``` ~/.everos///knowledge/ @@ -73,16 +73,16 @@ automatically by the cascade daemon. ### Storage roles ``` -Markdown (source of truth) + SQLite (structured state) + LanceDB (vector + BM25 index) +Markdown (source of truth) + SQLite (structured state) + derived vector/BM25 index ``` | Store | What it holds | Role | |-------|---------------|------| | Markdown | Document metadata, summaries, topic content, original files | Single source of truth; human-readable and editable | | SQLite | Document rows, topic rows (with content), change queue | Structured queries, paginated lists, count aggregation | -| LanceDB | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | +| Derived index | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | -Even if SQLite and LanceDB data is corrupted, as long as the Markdown +Even if SQLite and derived index data is corrupted, as long as the Markdown files are intact, the indexes can be fully rebuilt via the cascade daemon. ### Markdown format @@ -500,7 +500,7 @@ query ─→ embed ─→ keyword (BM25) ─┐ ``` 1. **Embed** — the query is embedded using the configured embedding provider -2. **Recall** — dual-channel retrieval from LanceDB: +2. **Recall** — dual-channel retrieval from the configured derived index: - BM25 channel: keyword matching on `summary_tokens` + `content_tokens` - ANN channel: nearest-neighbor search on the `vector` column - In `hybrid` mode, both channels run in parallel @@ -531,7 +531,7 @@ export EVEROS_KNOWLEDGE__SEARCH__RERANK_N=100 ## Cascade sync The cascade daemon watches the knowledge Markdown directory for file -changes and keeps SQLite + LanceDB in sync. +changes and keeps SQLite + the configured derived index in sync. ``` md file written @@ -539,7 +539,7 @@ md file written → worker picks up from queue (≤1s poll interval) → handler dispatched by file type: index.md → KnowledgeDocumentHandler → SQLite upsert (metadata) - N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + LanceDB upsert + N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + index upsert ``` The topic handler uses a SHA-256 content digest to skip unchanged files — @@ -597,7 +597,7 @@ curl .../documents?app_id=tenant_b&project_id=proj_1 # → { "documents": [], "total": 0 } ``` -Storage paths, SQLite rows, and LanceDB indexes are all scoped by +Storage paths, SQLite rows, and derived index rows are all scoped by `app_id` + `project_id`. ## End-to-end walkthrough diff --git a/docs/multimodal.md b/docs/multimodal.md index 0dc9e4ce3..e4fdcdfb4 100644 --- a/docs/multimodal.md +++ b/docs/multimodal.md @@ -41,7 +41,7 @@ POST /api/v1/memory/add boundary detector → extraction LLM → memory cell (MemCell) │ ▼ - markdown (truth) + SQLite (state) + LanceDB (vector + BM25) + markdown (truth) + SQLite (state) + derived vector/BM25 index │ ▼ retrievable via /search and /get like any text memory diff --git a/docs/overview.md b/docs/overview.md index 7f9be38ec..dccf68120 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -11,7 +11,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory - Local deployment for personal agents or small teams - Conversation, workflow, agent-trace, file-knowledge → structured memory - Hybrid retrieval (BM25 + vector + scalar filter) -- Cascade index sync (md edit → LanceDB sub-second) +- Cascade index sync (md edit → derived index) - Dual-track memory (user-track / agent-track) - Offline memory evolution (Foresight / AtomicFact / Profile / Skill), including Reflection — a consolidation strategy within the OME that @@ -30,7 +30,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory ### 1. Markdown as Source of Truth ``` -delete all LanceDB / SQLite files → can rebuild from md +delete all derived index / SQLite files → can rebuild from md delete any md file → memory is gone ``` @@ -42,7 +42,7 @@ User trust comes from physical visibility — the user can `cat` / `vim` / `grep |---|---|---| | Markdown files | Truth source — entries, frontmatter | Search (grep is degraded fallback only) | | SQLite | Queue, cascade audit log, sensitive data isolation | Vector / full-text | -| LanceDB | Vector ANN + BM25 + scalar filter, single-query hybrid | Be the source of truth (loss = rebuild from md) | +| Derived index | Vector ANN + BM25 + scalar filter | Be the source of truth (loss = rebuild from md) | ### 3. Algorithm-orchestration separation diff --git a/docs/reflection.md b/docs/reflection.md index 94a56aec3..620fcddd2 100644 --- a/docs/reflection.md +++ b/docs/reflection.md @@ -147,14 +147,15 @@ narrative. ## Storage layout -Memory uses Markdown as the single source of truth; SQLite and LanceDB are -derived indexes built automatically by the cascade daemon. +Memory uses Markdown as the single source of truth; SQLite and the configured +vector/BM25 index backend are derived indexes built automatically by the +cascade daemon. | Store | What it holds | Role | |---|---|---| | Markdown | Episode bodies, merged narratives, archive markers | Single source of truth; human-readable and editable | | SQLite | Clusters and members, consolidation audit records | Structured state and queries | -| LanceDB | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | +| Derived index | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | The **merged narrative** is written to the Episode daily-log Markdown; its frontmatter marks that it came from a cluster: @@ -186,7 +187,7 @@ deprecated_entries: --- ``` -> Soft-archive, not delete: even if SQLite / LanceDB are corrupted, as long +> Soft-archive, not delete: even if SQLite / the derived index are corrupted, as long > as the Markdown is intact the indexes can be fully rebuilt — and every > consolidation remains traceable back to its original content. @@ -355,5 +356,5 @@ curl -s -X POST "$BASE/memory/search" \ ## See also - [how-memory-works.md](how-memory-works.md) — Episodes and the memory extraction pipeline -- [storage_layout.md](storage_layout.md) — Markdown + SQLite + LanceDB stack +- [storage_layout.md](storage_layout.md) — Markdown + SQLite + derived index stack - [api.md](api.md) — full HTTP API reference diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 860446648..71447905a 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -3,8 +3,9 @@ How `everos` lays out a memory-root on disk: directory tree, file naming, frontmatter chassis, and entry-id encoding. -The contents are the **source of truth**; SQLite and LanceDB are -derived indexes that can be rebuilt from markdown alone. +The contents are the **source of truth**; SQLite and the configured +vector/BM25 index backend are derived indexes that can be rebuilt from +markdown alone. ## 1. Memory-root tree @@ -49,8 +50,10 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) -│ └── lancedb/ -│ └── .lance/ one directory per LanceDB table +│ ├── lancedb/ +│ │ └── .lance/ default derived index backend +│ └── milvus/ +│ └── milvus.db Milvus Lite database when configured │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -64,7 +67,7 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the -runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the +runtime-required dirs (`.index/{sqlite,lancedb,milvus}/`, `.tmp/`); the user-visible dirs are *not* pre-created — they appear on first write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. @@ -168,9 +171,9 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers > **File-level seq, not global**: the same `ep_20260601_00000001` may > appear across two different `user_id`s (each user has its own daily file). > Cross-table joins must therefore key on **`(scope_id, entry_id)`** -> rather than `entry_id` alone — see SQLite/LanceDB tables that follow. +> rather than `entry_id` alone — see the derived index tables that follow. -## 5. SQLite + LanceDB derived indexes +## 5. SQLite + derived indexes ``` .index/ @@ -178,9 +181,10 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ └── system.db state / audit / cascade queue + buffer / LSN │ (system tables: md_change_state, memcell, │ unprocessed_buffer, conversation_status, cluster) -└── lancedb/ - └── .lance/ one Arrow table per business kind — the per-kind - rows (text / vector / tokens / metadata) live here +├── lancedb/ +│ └── .lance/ default derived index backend +└── milvus/ + └── milvus.db Milvus Lite database when configured ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -190,13 +194,14 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers per-kind business rows. `reflection_report` is the audit trail for Reflection merges (cluster_id, mode, source_members, merged_entry_id, status). -- **LanceDB** ([`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/)) - holds the per-kind business rows, keyed `_` (so - cross-table joins use `(owner_id, entry_id)`); each table's `Vector(N)` - dimension matches the embedding model output. - -Episode and AtomicFact LanceDB tables carry a `deprecated_by: str | None` -column. When an episode is superseded by a Reflection merge, +- The **derived index backend** holds the per-kind business rows, keyed + `_` (so cross-table joins use `(owner_id, entry_id)`). + LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled + as the same rebuildable index backend and uses `.index/milvus/milvus.db` by + default for Milvus Lite. + +Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. +When an episode is superseded by a Reflection merge, `deprecated_by` is set to the merged episode's entry_id. Search filters automatically exclude rows where `deprecated_by IS NOT NULL`. @@ -226,5 +231,5 @@ this primitive is **schema-agnostic** — field-level semantics - Code: - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) - - [`infra/persistence/{markdown,sqlite,lancedb}/`](../src/everos/infra/persistence/) - - [`memory/cascade/`](../src/everos/memory/cascade/) (md → LanceDB sync) + - [`infra/persistence/{markdown,sqlite,lancedb,milvus,index}/`](../src/everos/infra/persistence/) + - [`memory/cascade/`](../src/everos/memory/cascade/) (md → derived index sync) diff --git a/pyproject.toml b/pyproject.toml index 0e359b6e6..86eac8594 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires-python = ">=3.12" authors = [ {name = "EverMind AI"}, ] -keywords = ["memory", "ai-agent", "markdown", "lancedb", "rag", "everos"] +keywords = ["memory", "ai-agent", "markdown", "lancedb", "milvus", "rag", "everos"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -26,7 +26,7 @@ dependencies = [ "pydantic-settings>=2.0.0", # Storage stack (md-first three-piece set) - "lancedb>=0.13.0", # Vector + BM25 + scalar filter (Arrow-based) + "lancedb>=0.13.0", # Default vector + BM25 + scalar filter index "aiosqlite>=0.20.0", # Async SQLite driver (used by SA async engine) "sqlmodel>=0.0.22", # ORM (Pydantic + SQLAlchemy 2.0 async) "alembic>=1.13.0", # SQLite schema migrations @@ -70,6 +70,9 @@ dependencies = [ [project.optional-dependencies] multimodal = ["everalgo-parser[svg]>=0.2.1"] # [svg] bundles cairosvg → SVG works by default +milvus = [ + "pymilvus[milvus-lite]>=3.0.0", +] [project.urls] Homepage = "https://evermind.ai" @@ -204,6 +207,7 @@ source_modules = ["everos.service", "everos.memory", "everos.entrypoints"] forbidden_modules = [ "everos.infra.persistence.markdown.**", "everos.infra.persistence.lancedb.**", + "everos.infra.persistence.milvus.**", "everos.infra.persistence.sqlite.**", ] # `forbidden` contracts run a *transitive closure* — any path from a @@ -222,12 +226,17 @@ forbidden_modules = [ # in ``service`` or ``entrypoints``) remain blocked — that edge is not # in this whitelist. ignore_imports = [ + "everos.infra.persistence.index -> everos.infra.persistence.lancedb", + "everos.infra.persistence.index -> everos.infra.persistence.milvus", "everos.infra.persistence.sqlite -> everos.infra.persistence.sqlite.tables", "everos.infra.persistence.sqlite -> everos.infra.persistence.sqlite.repos", "everos.infra.persistence.sqlite -> everos.infra.persistence.sqlite.sqlite_manager", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.tables", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.repos", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.lancedb_manager", + "everos.infra.persistence.milvus -> everos.infra.persistence.milvus.repos", + "everos.infra.persistence.milvus -> everos.infra.persistence.milvus.milvus_manager", + "everos.infra.persistence.milvus.repos -> everos.infra.persistence.milvus.repository", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", diff --git a/src/everos/README.md b/src/everos/README.md index 8cc1cef96..2be6caf25 100644 --- a/src/everos/README.md +++ b/src/everos/README.md @@ -10,7 +10,7 @@ everos/ ├── entrypoints/ Presentation: cli + api ├── service/ Application: use case orchestration ├── memory/ Domain: extract + search + cascade + prompt_slots + models -├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb} +├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb, milvus, index} ├── component/ Cross-cutting providers: llm / embedding / config / utils ├── core/ Runtime base: observability / lifespan / context └── config/ Data: Settings + default.toml + prompt_slots templates diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index c29590230..7ff8db9c9 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -2,7 +2,8 @@ Public API: from everos.config import ( - Settings, MemorySettings, SqliteSettings, LanceDBSettings, + Settings, MemorySettings, SqliteSettings, IndexSettings, LanceDBSettings, + MilvusSettings, LLMSettings, EmbeddingSettings, RerankSettings, BoundaryDetectionSettings, load_settings, resolve_root, @@ -14,9 +15,11 @@ from .settings import BoundaryDetectionSettings as BoundaryDetectionSettings from .settings import EmbeddingSettings as EmbeddingSettings +from .settings import IndexSettings as IndexSettings from .settings import LanceDBSettings as LanceDBSettings from .settings import LLMSettings as LLMSettings from .settings import MemorySettings as MemorySettings +from .settings import MilvusSettings as MilvusSettings from .settings import MultimodalSettings as MultimodalSettings from .settings import RerankSettings as RerankSettings from .settings import Settings as Settings @@ -27,9 +30,11 @@ __all__ = [ "BoundaryDetectionSettings", "EmbeddingSettings", + "IndexSettings", "LLMSettings", "LanceDBSettings", "MemorySettings", + "MilvusSettings", "MultimodalSettings", "RerankSettings", "Settings", diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index 68fab81f1..9034ebcdf 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -50,6 +50,24 @@ cache_size_kb = 2048 # Uncomment to override: # read_consistency_seconds = 5.0 +[index] +# Rebuildable vector/BM25 index backend. Markdown remains the source of truth +# and SQLite remains the system state store. +# Override via EVEROS_INDEX__BACKEND. +backend = "lancedb" + +[milvus] +# Optional derived index backend. Leave uri empty to use Milvus Lite at +# /.index/milvus/milvus.db. Set uri/token/db_name for Milvus server or +# Zilliz Cloud. +# Override via EVEROS_MILVUS__URI, EVEROS_MILVUS__TOKEN, EVEROS_MILVUS__DB_NAME. +uri = "" +token = "" +db_name = "" +consistency_level = "Session" +dimension = 1024 +collection_prefix = "everos" + [llm] # Provider-agnostic OpenAI-protocol client config. Override via env: # EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL @@ -144,4 +162,3 @@ session_lock_timeout_seconds = 360.0 # Override via EVEROS_CLUSTERING__THRESHOLD, EVEROS_CLUSTERING__TIME_WINDOW_DAYS. threshold = 0.65 time_window_days = 7.0 - diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 5ae2c8da8..8d5c593ba 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -331,6 +331,47 @@ class LanceDBSettings(BaseModel): index_cache_size_bytes: int = 16 * 1024 * 1024 +class IndexSettings(BaseModel): + """Derived index backend selection. + + Markdown remains the source of truth and SQLite remains the system + state store. This setting only selects the rebuildable vector/BM25 + index used by cascade and search. + + Env binding: + EVEROS_INDEX__BACKEND + """ + + backend: Literal["lancedb", "milvus"] = "lancedb" + + +class MilvusSettings(BaseModel): + """Milvus derived index connection and schema settings. + + ``uri``: + ``None`` or empty string uses Milvus Lite under the memory root + (``/.index/milvus/milvus.db``). Set to a Milvus server or + Zilliz Cloud endpoint to use an external service. + + Env binding: + EVEROS_MILVUS__URI + EVEROS_MILVUS__TOKEN + EVEROS_MILVUS__DB_NAME + EVEROS_MILVUS__CONSISTENCY_LEVEL + EVEROS_MILVUS__DIMENSION + EVEROS_MILVUS__COLLECTION_PREFIX + """ + + uri: str | None = None + token: SecretStr | None = None + db_name: str | None = None + consistency_level: Literal["Strong", "Session", "Bounded", "Eventually"] = ( + "Session" + ) + dimension: int = Field(default=1024, ge=1) + collection_prefix: str = "everos" + + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" @@ -356,7 +397,9 @@ class Settings(BaseSettings): memory: MemorySettings = MemorySettings() api: ApiSettings = ApiSettings() sqlite: SqliteSettings = SqliteSettings() + index: IndexSettings = IndexSettings() lancedb: LanceDBSettings = LanceDBSettings() + milvus: MilvusSettings = MilvusSettings() llm: LLMSettings = LLMSettings() embedding: EmbeddingSettings = EmbeddingSettings() rerank: RerankSettings = RerankSettings() diff --git a/src/everos/core/persistence/memory_root.py b/src/everos/core/persistence/memory_root.py index 7a96bd2f9..efa507630 100644 --- a/src/everos/core/persistence/memory_root.py +++ b/src/everos/core/persistence/memory_root.py @@ -11,6 +11,7 @@ .index/ derived indexes (rebuildable from markdown) sqlite/ system.db (+ WAL/SHM), ome.db, ome.aps.db lancedb/ LanceDB tables + milvus/ Milvus Lite database when Milvus is selected .tmp/ atomic-write staging directory .lock single-process lock anchor (created on demand by ``memory_root_lock``) @@ -145,6 +146,16 @@ def lancedb_dir(self) -> Path: """``/.index/lancedb/`` — LanceDB table root.""" return self.index_dir / "lancedb" + @property + def milvus_dir(self) -> Path: + """``/.index/milvus/`` — Milvus Lite derived index root.""" + return self.index_dir / "milvus" + + @property + def milvus_db(self) -> Path: + """``/.index/milvus/milvus.db`` — default Milvus Lite URI.""" + return self.milvus_dir / "milvus.db" + @property def sqlite_dir(self) -> Path: """``/.index/sqlite/`` — SQLite system DB root. @@ -224,10 +235,12 @@ def ensure(self) -> None: /.index/ /.index/sqlite/ /.index/lancedb/ + /.index/milvus/ /.tmp/ """ self.root.mkdir(parents=True, exist_ok=True) self.index_dir.mkdir(parents=True, exist_ok=True) self.sqlite_dir.mkdir(parents=True, exist_ok=True) self.lancedb_dir.mkdir(parents=True, exist_ok=True) + self.milvus_dir.mkdir(parents=True, exist_ok=True) self.tmp_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index b2a030bdc..0a6b00082 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -1,13 +1,11 @@ -"""LanceDB lifespan provider (HTTP API entrypoint). +"""Derived index lifespan provider (HTTP API entrypoint). Startup: - Open the connection via ``get_connection`` (lazy, idempotent). - Importing :mod:`everos.infra.persistence.lancedb` also triggers the - side-effect import of ``tables`` so business schemas are loaded - (future: preflight registration). + Start the configured derived index backend (LanceDB by default, + Milvus when ``[index] backend = "milvus"``). Shutdown: - Close the connection (also clears the table cache). + Close the active backend connection. """ from __future__ import annotations @@ -18,38 +16,24 @@ from everos.core.lifespan import LifespanProvider from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import ( - dispose_connection, - ensure_business_indexes, - get_connection, - verify_business_schemas, -) +from everos.infra.persistence.index import active_backend, shutdown, startup logger = get_logger(__name__) class LanceDBLifespanProvider(LifespanProvider): - """Manage the LanceDB connection + table cache for the app lifecycle. + """Manage the derived index backend for the app lifecycle. - Startup runs three steps: - - 1. ``get_connection`` — lazy-open the async connection. - 2. ``verify_business_schemas`` — fail loud if an on-disk table's - columns drift from the current Pydantic schema. LanceDB has no - online migration; cascade is rebuildable from md so the recovery - is documented as ``rm -rf ~/.everos/.index/lancedb``. - 3. ``ensure_business_indexes`` — idempotent FTS index creation. + The class name is kept for compatibility with existing app wiring. """ def __init__(self, order: int = 11) -> None: - super().__init__(name="lancedb", order=order) + super().__init__(name="derived_index", order=order) async def startup(self, app: FastAPI) -> Any: - conn = await get_connection() - await verify_business_schemas() - await ensure_business_indexes() - logger.info("lancedb_ready", uri=conn.uri) - return conn + resource = await startup() + logger.info("derived_index_ready", backend=active_backend()) + return resource async def shutdown(self, app: FastAPI) -> None: - await dispose_connection() + await shutdown() diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 8d27e8aa0..328e25239 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -34,12 +34,8 @@ from everos.component.utils.datetime import to_display_tz from everos.config import load_settings from everos.core.persistence import MemoryRoot -from everos.infra.persistence.lancedb import ( - dispose_connection, - ensure_business_indexes, - get_connection, - verify_business_schemas, -) +from everos.infra.persistence.index import shutdown as shutdown_index +from everos.infra.persistence.index import startup as startup_index from everos.infra.persistence.sqlite import ( dispose_engine, get_engine, @@ -49,7 +45,7 @@ app = typer.Typer( name="cascade", - help="Inspect and operate the md → LanceDB sync queue", + help="Inspect and operate the md → derived index sync queue", no_args_is_help=True, ) @@ -72,7 +68,7 @@ def _cascade_callback( @asynccontextmanager async def _runtime(): # type: ignore[no-untyped-def] - """Stand up sqlite + lancedb the same way the API lifespan would. + """Stand up sqlite + the derived index the same way the API lifespan would. The CLI piggybacks on the same singletons as the running daemon (lazy + process-wide), so if a server happens to be running on @@ -81,13 +77,11 @@ async def _runtime(): # type: ignore[no-untyped-def] engine = get_engine() async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) - await get_connection() - await verify_business_schemas() - await ensure_business_indexes() + await startup_index() try: yield finally: - await dispose_connection() + await shutdown_index() await dispose_engine() diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py new file mode 100644 index 000000000..486900058 --- /dev/null +++ b/src/everos/infra/persistence/index/__init__.py @@ -0,0 +1,329 @@ +"""Backend-neutral derived index persistence facade. + +Markdown is the source of truth and SQLite stores system state. This package +selects the rebuildable vector/BM25 index backend used by cascade, search, and +get. The default backend remains LanceDB; Milvus is opt-in through +``[index] backend = "milvus"``. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery + +try: + from lancedb.query import Occur +except ImportError: # pragma: no cover + from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] + +from everos.config import load_settings +from everos.infra.persistence import lancedb as _lancedb +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + ParentType, + UserProfile, +) + + +def active_backend() -> str: + """Return the configured derived index backend name.""" + return load_settings().index.backend + + +async def startup() -> Any: + """Initialise the configured derived index backend.""" + if active_backend() == "milvus": + milvus = _milvus() + await milvus.get_client() + await milvus.ensure_business_indexes() + return "milvus" + conn = await _lancedb.get_connection() + await _lancedb.verify_business_schemas() + await _lancedb.ensure_business_indexes() + return conn + + +async def shutdown() -> None: + """Dispose the configured derived index backend.""" + if active_backend() == "milvus": + await _milvus().dispose_connection() + else: + await _lancedb.dispose_connection() + + +async def ensure_business_indexes() -> None: + if active_backend() == "milvus": + await _milvus().ensure_business_indexes() + else: + await _lancedb.ensure_business_indexes() + + +async def verify_business_schemas() -> None: + if active_backend() == "milvus": + await _milvus().verify_business_schemas() + else: + await _lancedb.verify_business_schemas() + + +class _LanceIndexRepoAdapter: + """Add backend-neutral recall helpers to an existing LanceDB repo.""" + + def __init__(self, repo: Any, schema: type[Any]) -> None: + self._repo = repo + self.schema = schema + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo, name) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + fields = list(columns or self.schema.BM25_FIELDS) + if not query_terms or not fields: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + expr = _lance_expr(where) + best: dict[str, dict[str, Any]] = {} + for field in fields: + query = _build_or_query(query_terms, field) + rows = ( + await table.query() + .nearest_to_text(query) + .where(expr) + .limit(limit) + .to_list() + ) + for row in rows: + rid = row.get("id") + if not isinstance(rid, str): + continue + score = float(row.get("_score", 0.0)) + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + shaped = dict(row) + shaped["_score"] = score + best[rid] = shaped + return sorted( + best.values(), key=lambda row: float(row.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + ) -> list[dict[str, Any]]: + if not vector: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + return ( + await table.query() + .nearest_to(list(vector)) + .distance_type("cosine") + .where(_lance_expr(where)) + .limit(limit) + .to_list() + ) + + +class _IndexRepoRouter: + """Route repo calls to the configured derived index backend.""" + + def __init__( + self, lance_repo: Any, milvus_repo_name: str, schema: type[Any] + ) -> None: + self._lance = _LanceIndexRepoAdapter(lance_repo, schema) + self._milvus_repo_name = milvus_repo_name + self.schema = schema + + @property + def table_name(self) -> str: + return self.schema.TABLE_NAME + + def _repo(self) -> Any: + if active_backend() == "milvus": + return getattr(_milvus(), self._milvus_repo_name) + return self._lance + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo(), name) + + async def add(self, records: Sequence[Any]) -> None: + await self._repo().add(records) + + async def upsert(self, records: Sequence[Any], *, by: str = "id") -> None: + await self._repo().upsert(records, by=by) + + async def count(self) -> int: + return await self._repo().count() + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> Any: + return await self._repo().get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Any, *, limit: int = 100) -> list[Any]: + return await self._repo().find_where(_where_for_backend(where), limit=limit) + + async def find_one_where(self, where: Any) -> Any: + return await self._repo().find_one_where(_where_for_backend(where)) + + async def find_where_paginated( + self, + where: Any, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[Any], int]: + return await self._repo().find_where_paginated( + _where_for_backend(where), + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Any = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + return await self._repo().search( + vector=vector, where=_where_for_backend(where), limit=limit + ) + + async def update(self, updates: dict[str, Any], *, where: Any) -> None: + await self._repo().update(updates, where=_where_for_backend(where)) + + async def delete(self, predicate: Any) -> None: + await self._repo().delete(_where_for_backend(predicate)) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo().delete_by_md_path(md_path) + + async def optimize(self, **kwargs: Any) -> None: + await self._repo().optimize(**kwargs) + + async def rebuild_indexes(self) -> None: + await self._repo().rebuild_indexes() + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + return await self._repo().sparse_search( + query_terms, + _where_for_backend(where), + columns=columns, + limit=limit, + ) + + async def dense_search( + self, vector: Sequence[float], where: Any, *, limit: int + ) -> list[dict[str, Any]]: + return await self._repo().dense_search( + vector, _where_for_backend(where), limit=limit + ) + + +def _where_for_backend(where: Any) -> Any: + if active_backend() == "milvus": + return getattr(where, "milvus", where) + return _lance_expr(where) + + +def _lance_expr(where: Any) -> str: + return getattr(where, "lancedb", where) + + +def _build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery: + clean = [token for token in tokens if token] + if not clean: + return MatchQuery("", column=column) + if len(clean) == 1: + return MatchQuery(clean[0], column=column) + return BooleanQuery( + [(Occur.SHOULD, MatchQuery(token, column=column)) for token in clean] + ) + + +def _milvus() -> Any: + from everos.infra.persistence import milvus + + return milvus + + +episode_repo = _IndexRepoRouter(_lancedb.episode_repo, "episode_repo", Episode) +atomic_fact_repo = _IndexRepoRouter( + _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact +) +foresight_repo = _IndexRepoRouter( + _lancedb.foresight_repo, "foresight_repo", Foresight +) +agent_case_repo = _IndexRepoRouter( + _lancedb.agent_case_repo, "agent_case_repo", AgentCase +) +agent_skill_repo = _IndexRepoRouter( + _lancedb.agent_skill_repo, "agent_skill_repo", AgentSkill +) +user_profile_repo = _IndexRepoRouter( + _lancedb.user_profile_repo, "user_profile_repo", UserProfile +) +knowledge_topic_repo = _IndexRepoRouter( + _lancedb.knowledge_topic_repo, "knowledge_topic_repo", KnowledgeTopic +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "AgentCase", + "AgentSkill", + "AtomicFact", + "Episode", + "Foresight", + "KnowledgeTopic", + "ParentType", + "UserProfile", + "active_backend", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "shutdown", + "startup", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py new file mode 100644 index 000000000..4396360c8 --- /dev/null +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -0,0 +1,49 @@ +"""Milvus derived index backend. + +This package mirrors the LanceDB business index surface but stores rows in +Milvus collections. It is selected through ``Settings.index.backend`` and is +normally reached through :mod:`everos.infra.persistence.index`. +""" + +from __future__ import annotations + +from .milvus_manager import MilvusSchemaMismatchError as MilvusSchemaMismatchError +from .milvus_manager import dispose_connection as dispose_connection +from .milvus_manager import get_client as get_client +from .repos import ALL_REPOS as ALL_REPOS +from .repos import agent_case_repo as agent_case_repo +from .repos import agent_skill_repo as agent_skill_repo +from .repos import atomic_fact_repo as atomic_fact_repo +from .repos import episode_repo as episode_repo +from .repos import foresight_repo as foresight_repo +from .repos import knowledge_topic_repo as knowledge_topic_repo +from .repos import user_profile_repo as user_profile_repo + + +async def ensure_business_indexes() -> None: + """Create or verify every EverOS Milvus collection.""" + for repo in ALL_REPOS: + await repo.ensure_collection() + + +async def verify_business_schemas() -> None: + """Fail loud if existing Milvus collections drift from EverOS schemas.""" + for repo in ALL_REPOS: + await repo.verify_collection() + + +__all__ = [ + "ALL_REPOS", + "MilvusSchemaMismatchError", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "dispose_connection", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "get_client", + "knowledge_topic_repo", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/milvus/milvus_manager.py b/src/everos/infra/persistence/milvus/milvus_manager.py new file mode 100644 index 000000000..602e4a314 --- /dev/null +++ b/src/everos/infra/persistence/milvus/milvus_manager.py @@ -0,0 +1,87 @@ +"""Milvus connection and collection management for the derived index.""" + +from __future__ import annotations + +import re + +from pymilvus import MilvusClient + +from everos.config import MilvusSettings, load_settings +from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot + +logger = get_logger(__name__) + +_client: MilvusClient | None = None + + +class MilvusSchemaMismatchError(RuntimeError): + """Raised when an existing Milvus collection does not match EverOS.""" + + +def collection_name(table_name: str, settings: MilvusSettings | None = None) -> str: + """Return the configured Milvus collection name for an EverOS table.""" + cfg = settings or load_settings().milvus + prefix = _sanitize_name_part(cfg.collection_prefix) + base = _sanitize_name_part(table_name) + name = f"{prefix}_{base}" if prefix else base + if not re.match(r"^[A-Za-z_]", name): + name = f"_{name}" + return name + + +async def get_client() -> MilvusClient: + """Return the process-wide MilvusClient, creating it lazily.""" + global _client + if _client is None: + settings = load_settings().milvus + uri = _resolve_uri(settings) + token = _secret(settings.token) + db_name = settings.db_name or "" + _client = MilvusClient(uri=uri, token=token, db_name=db_name) + logger.info( + "milvus_connection_opened", + uri=uri, + db_name=db_name or None, + consistency_level=settings.consistency_level, + ) + return _client + + +async def dispose_connection() -> None: + """Close the process-wide Milvus client.""" + global _client + if _client is not None: + _client.close() + _client = None + logger.info("milvus_connection_closed") + + +def _resolve_uri(settings: MilvusSettings) -> str: + if settings.uri: + return settings.uri + memory_root = MemoryRoot.default() + memory_root.milvus_dir.mkdir(parents=True, exist_ok=True) + return str(memory_root.milvus_db) + + +def _secret(value: object | None) -> str: + if value is None: + return "" + getter = getattr(value, "get_secret_value", None) + if callable(getter): + return getter() or "" + return str(value) + + +def _sanitize_name_part(value: str) -> str: + clean = re.sub(r"\W+", "_", value.strip()) + return clean.strip("_") + + +__all__ = [ + "MilvusSchemaMismatchError", + "collection_name", + "dispose_connection", + "get_client", +] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py new file mode 100644 index 000000000..826bc9267 --- /dev/null +++ b/src/everos/infra/persistence/milvus/repos.py @@ -0,0 +1,119 @@ +"""Milvus repo singletons for EverOS derived index tables.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + UserProfile, +) + +from .repository import MilvusRepoBase + + +class _EpisodeRepo(MilvusRepoBase[Episode]): + schema = Episode + + +class _AtomicFactRepo(MilvusRepoBase[AtomicFact]): + schema = AtomicFact + + +class _ForesightRepo(MilvusRepoBase[Foresight]): + schema = Foresight + + +class _AgentCaseRepo(MilvusRepoBase[AgentCase]): + schema = AgentCase + + +class _AgentSkillRepo(MilvusRepoBase[AgentSkill]): + schema = AgentSkill + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + rows = await self.find_where( + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=10_000, + ) + return len(rows) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[AgentSkill]: + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[AgentSkill]: + if not query_vector: + raise ValueError( + "query_vector must be non-empty; " + "call find_in_cluster for the scalar fallback" + ) + rows = await self.dense_search( + query_vector, + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=top_k, + ) + out: list[AgentSkill] = [] + for row in rows: + rid = row.get("id") + if isinstance(rid, str) and (item := await self.get_by_id(rid)) is not None: + out.append(item) + return out + + +class _UserProfileRepo(MilvusRepoBase[UserProfile]): + schema = UserProfile + + +class _KnowledgeTopicRepo(MilvusRepoBase[KnowledgeTopic]): + schema = KnowledgeTopic + + +def _q(value: str) -> str: + return value.replace("\\", "\\\\").replace("'", "\\'") + + +episode_repo = _EpisodeRepo() +atomic_fact_repo = _AtomicFactRepo() +foresight_repo = _ForesightRepo() +agent_case_repo = _AgentCaseRepo() +agent_skill_repo = _AgentSkillRepo() +user_profile_repo = _UserProfileRepo() +knowledge_topic_repo = _KnowledgeTopicRepo() + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py new file mode 100644 index 000000000..b8828118e --- /dev/null +++ b/src/everos/infra/persistence/milvus/repository.py @@ -0,0 +1,750 @@ +"""Milvus repository implementation for EverOS derived index tables.""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import re +from collections.abc import Sequence +from importlib.metadata import PackageNotFoundError, version +from typing import Any, ClassVar, get_args, get_origin + +from pymilvus import DataType, Function, FunctionType, MilvusClient + +from everos.component.utils.datetime import ( + ensure_utc, + from_iso_format, + from_timestamp, + to_timestamp_ms, +) +from everos.config import load_settings +from everos.core.observability.logging import get_logger +from everos.core.persistence import BaseLanceTable + +from .milvus_manager import ( + MilvusSchemaMismatchError, + collection_name, + get_client, +) + +logger = get_logger(__name__) + +_DUMMY_VECTOR_FIELD = "_everos_dummy_vector" +_SPARSE_SUFFIX = "__sparse" +_MAX_VARCHAR_LENGTH = 65_535 +_ID_MAX_LENGTH = 512 +_ARRAY_MAX_CAPACITY = 256 + + +def _q(value: str) -> str: + """Escape a string for a Milvus single-quoted expression literal.""" + return value.replace("\\", "\\\\").replace("'", "\\'") + + +class MilvusRepoBase[T: BaseLanceTable]: + """Generic Milvus repository for one EverOS derived index collection.""" + + schema: type[T] + _write_locks: ClassVar[dict[str, asyncio.Lock]] = {} + + @property + def table_name(self) -> str: + return self.schema.TABLE_NAME + + @property + def collection_name(self) -> str: + return collection_name(self.table_name) + + @classmethod + def _write_lock(cls, name: str) -> asyncio.Lock: + return cls._write_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_locks_for_tests(cls) -> None: + cls._write_locks.clear() + + async def ensure_collection(self) -> None: + client = await get_client() + if await _run(client.has_collection, self.collection_name): + await self.verify_collection() + return + + schema = self._build_collection_schema() + index_params = client.prepare_index_params() + if "vector" in self.schema.model_fields: + index_params.add_index( + field_name="vector", + index_type="AUTOINDEX", + metric_type="COSINE", + ) + else: + index_params.add_index( + field_name=_DUMMY_VECTOR_FIELD, + index_type="AUTOINDEX", + metric_type="COSINE", + ) + for field in self.schema.BM25_FIELDS: + index_params.add_index( + field_name=_sparse_field(field), + index_type="AUTOINDEX", + metric_type="BM25", + ) + + settings = load_settings().milvus + await _run( + client.create_collection, + collection_name=self.collection_name, + schema=schema, + index_params=index_params, + consistency_level=settings.consistency_level, + ) + logger.info( + "milvus_collection_created", + table=self.table_name, + collection=self.collection_name, + ) + + async def verify_collection(self) -> None: + client = await get_client() + description = await _run(client.describe_collection, self.collection_name) + actual = {field["name"] for field in description.get("fields", [])} + expected = set(self._stored_field_names()) + missing = expected - actual + if missing: + raise MilvusSchemaMismatchError( + f"Milvus collection {self.collection_name!r} schema drift: " + f"missing={sorted(missing)}. The index is rebuildable from md; " + "drop the collection and restart to rebuild it." + ) + + # ── Create / update ───────────────────────────────────────────── + + async def add(self, records: Sequence[T]) -> None: + if not records: + return + await self.ensure_collection() + client = await get_client() + payload = [self._to_milvus_record(record) for record in records] + async with self._write_lock(self.collection_name): + await _run(client.insert, self.collection_name, payload) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + if by != "id": + raise ValueError("MilvusRepoBase only supports upsert by id") + if not records: + return + await self.ensure_collection() + client = await get_client() + payload = [self._to_milvus_record(record) for record in records] + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, payload) + + async def update(self, updates: dict[str, Any], *, where: Any) -> None: + await self.ensure_collection() + rows = await self._query_raw(where, limit=10_000, include_vectors=True) + if not rows: + return + patched = [] + for row in rows: + merged = dict(row) + for key, value in updates.items(): + self._write_field_value(merged, key, value) + patched.append(merged) + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, patched) + + # ── Maintenance ──────────────────────────────────────────────── + + async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: + """Milvus indexes are managed by the service; no per-write compaction.""" + + async def rebuild_indexes(self) -> None: + """Milvus AUTOINDEX maintenance is service-managed.""" + + # ── Reads ────────────────────────────────────────────────────── + + async def count(self) -> int: + await self.ensure_collection() + client = await get_client() + stats = await _run(client.get_collection_stats, self.collection_name) + row_count = stats.get("row_count", 0) + return int(row_count) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + if id_field != "id": + rows = await self.find_where(f"{id_field} = '{_q(id_value)}'", limit=1) + return rows[0] if rows else None + await self.ensure_collection() + client = await get_client() + rows = await _run( + client.get, + self.collection_name, + ids=[id_value], + output_fields=self._output_fields(include_vectors=True), + ) + if not rows: + return None + return self._model_from_milvus(rows[0]) + + async def find_where(self, where: Any, *, limit: int = 100) -> list[T]: + rows = await self._query_raw(where, limit=limit, include_vectors=True) + return [self._model_from_milvus(row) for row in rows] + + async def find_one_where(self, where: Any) -> T | None: + rows = await self.find_where(where, limit=1) + return rows[0] if rows else None + + async def find_where_paginated( + self, + where: Any, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + raw = await self._query_raw(where, limit=max_fetch, include_vectors=True) + total = len(raw) + if total >= max_fetch: + logger.warning( + "milvus_find_where_paginated_truncated", + table=self.table_name, + total=total, + max_fetch=max_fetch, + ) + rows = [self._model_from_milvus(row) for row in raw] + rows.sort( + key=lambda row: _sort_value(getattr(row, sort_by, None)), + reverse=descending, + ) + offset = (page - 1) * page_size + return rows[offset : offset + page_size], total + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(f"owner_id = '{_q(owner_id)}'", limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(f"md_path = '{_q(md_path)}'") + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Any = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + if vector is None: + return await self._query_candidate_rows(where, limit=limit) + return await self.dense_search(vector, where, limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + if not query_terms: + return [] + await self.ensure_collection() + fields = list(columns or self.schema.BM25_FIELDS) + if not fields: + return [] + client = await get_client() + expr = self._expr(where) + query = " ".join(term for term in query_terms if term) + best: dict[str, dict[str, Any]] = {} + for field in fields: + results = await _run( + client.search, + self.collection_name, + data=[query], + anns_field=_sparse_field(field), + filter=expr, + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "BM25"}, + ) + for row in _first_result_set(results): + shaped = self._candidate_row_from_search(row) + score = _bm25_score_from_distance(row.get("distance")) + shaped["_score"] = score + rid = shaped.get("id") + if not isinstance(rid, str): + continue + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + best[rid] = shaped + return sorted( + best.values(), key=lambda item: float(item.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + ) -> list[dict[str, Any]]: + if not vector or "vector" not in self.schema.model_fields: + return [] + await self.ensure_collection() + client = await get_client() + results = await _run( + client.search, + self.collection_name, + data=[list(vector)], + anns_field="vector", + filter=self._expr(where), + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "COSINE"}, + ) + return [ + self._candidate_row_from_search(row, normalize_cosine=True) + for row in _first_result_set(results) + ] + + # ── Deletes ──────────────────────────────────────────────────── + + async def delete(self, predicate: Any) -> None: + await self.ensure_collection() + client = await get_client() + async with self._write_lock(self.collection_name): + await _run( + client.delete, + self.collection_name, + filter=self._expr(predicate), + ) + + async def delete_by_md_path(self, md_path: str) -> int: + await self.ensure_collection() + before = await self.find_where(f"md_path = '{_q(md_path)}'", limit=10_000) + if not before: + return 0 + await self.delete(f"md_path = '{_q(md_path)}'") + return len(before) + + # ── Daily-log helpers ────────────────────────────────────────── + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + f"owner_id = '{_q(owner_id)}' AND entry_id = '{_q(entry_id)}' " + f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'" + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + quoted = ", ".join(f"'{_q(entry_id)}'" for entry_id in entry_ids) + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND entry_id IN ({quoted}) " + f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'", + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND session_id = '{_q(session_id)}'", + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + f"parent_type = '{_q(parent_type)}' AND parent_id = '{_q(parent_id)}'", + limit=limit, + ) + + # ── Field conversion ─────────────────────────────────────────── + + def _build_collection_schema(self): # type: ignore[no-untyped-def] + settings = load_settings().milvus + schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) + has_vector = False + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + annotation = field.annotation + if name == "id": + schema.add_field( + field_name=name, + datatype=DataType.VARCHAR, + is_primary=True, + max_length=_ID_MAX_LENGTH, + ) + elif name == "vector": + has_vector = True + schema.add_field( + field_name=name, + datatype=DataType.FLOAT_VECTOR, + dim=settings.dimension, + ) + elif _is_datetime(annotation): + schema.add_field( + field_name=_datetime_storage_field(name), + datatype=DataType.INT64, + nullable=_is_optional(annotation), + ) + elif _is_list(annotation): + schema.add_field( + field_name=name, + datatype=DataType.ARRAY, + element_type=DataType.VARCHAR, + max_capacity=_ARRAY_MAX_CAPACITY, + max_length=_ID_MAX_LENGTH, + nullable=_is_optional(annotation), + ) + elif _is_float(annotation): + schema.add_field( + field_name=name, + datatype=DataType.DOUBLE, + nullable=_is_optional(annotation), + ) + elif _is_int(annotation): + schema.add_field( + field_name=name, + datatype=DataType.INT64, + nullable=_is_optional(annotation), + ) + else: + kwargs: dict[str, Any] = { + "field_name": name, + "datatype": DataType.VARCHAR, + "max_length": _MAX_VARCHAR_LENGTH, + } + if name in self.schema.BM25_FIELDS: + kwargs["enable_analyzer"] = True + elif _is_optional(annotation): + kwargs["nullable"] = True + schema.add_field(**kwargs) + + if not has_vector: + schema.add_field( + field_name=_DUMMY_VECTOR_FIELD, + datatype=DataType.FLOAT_VECTOR, + dim=1, + ) + for field in self.schema.BM25_FIELDS: + schema.add_field( + field_name=_sparse_field(field), + datatype=DataType.SPARSE_FLOAT_VECTOR, + ) + schema.add_function( + Function( + name=f"{field}_bm25", + function_type=FunctionType.BM25, + input_field_names=[field], + output_field_names=[_sparse_field(field)], + ) + ) + return schema + + def _stored_field_names(self) -> list[str]: + names: list[str] = [] + has_vector = False + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + if name == "vector": + has_vector = True + names.append(name) + elif _is_datetime(field.annotation): + names.append(_datetime_storage_field(name)) + else: + names.append(name) + if not has_vector: + names.append(_DUMMY_VECTOR_FIELD) + names.extend(_sparse_field(field) for field in self.schema.BM25_FIELDS) + return names + + def _output_fields(self, *, include_vectors: bool) -> list[str]: + fields: list[str] = [] + for name in self._stored_field_names(): + if name.endswith(_SPARSE_SUFFIX) or name == _DUMMY_VECTOR_FIELD: + continue + if not include_vectors and name == "vector": + continue + fields.append(name) + return fields + + def _to_milvus_record(self, record: T) -> dict[str, Any]: + raw = record.model_dump(mode="python") + out: dict[str, Any] = {} + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + value = raw.get(name) + if name in self.schema.BM25_FIELDS and value is None: + value = "" + if _is_datetime(field.annotation): + storage_name = _datetime_storage_field(name) + out[storage_name] = ( + _datetime_to_ms(value) if value is not None else None + ) + elif name == "vector": + out[name] = list(value or []) + elif _is_list(field.annotation): + out[name] = [str(item) for item in (value or [])] + else: + out[name] = value + if "vector" not in self.schema.model_fields: + out[_DUMMY_VECTOR_FIELD] = [0.0] + return out + + def _model_from_milvus(self, row: dict[str, Any]) -> T: + shaped = self._restore_row(row, include_distance=False) + return self.schema.model_validate(shaped) + + def _candidate_row_from_search( + self, row: dict[str, Any], *, normalize_cosine: bool = False + ) -> dict[str, Any]: + shaped = self._restore_row(row.get("entity", {}), include_distance=False) + raw_distance = row.get("distance") + shaped["_distance"] = ( + _cosine_distance_from_milvus(raw_distance) + if normalize_cosine + else raw_distance + ) + return shaped + + def _restore_row( + self, row: dict[str, Any], *, include_distance: bool + ) -> dict[str, Any]: + out: dict[str, Any] = {} + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + if _is_datetime(field.annotation): + storage_name = _datetime_storage_field(name) + value = row.get(storage_name) + if value is None: + out[name] = None + else: + out[name] = from_timestamp(int(value)) + elif name in row: + value = row[name] + if name in self.schema.BM25_FIELDS and value == "" and _is_optional( + field.annotation + ): + out[name] = None + else: + out[name] = value + if include_distance and "distance" in row: + out["_distance"] = row["distance"] + return out + + def _write_field_value( + self, row: dict[str, Any], field_name: str, value: Any + ) -> None: + field = self.schema.model_fields.get(field_name) + if field is None: + row[field_name] = value + elif _is_datetime(field.annotation): + row[_datetime_storage_field(field_name)] = ( + _datetime_to_ms(value) if value is not None else None + ) + else: + row[field_name] = value + + async def _query_raw( + self, where: Any, *, limit: int, include_vectors: bool + ) -> list[dict[str, Any]]: + await self.ensure_collection() + client = await get_client() + return await _run( + client.query, + self.collection_name, + filter=self._expr(where), + output_fields=self._output_fields(include_vectors=include_vectors), + limit=limit, + ) + + async def _query_candidate_rows( + self, where: Any, *, limit: int + ) -> list[dict[str, Any]]: + rows = await self._query_raw(where, limit=limit, include_vectors=False) + return [self._restore_row(row, include_distance=False) for row in rows] + + def _expr(self, where: Any) -> str: + if where is None: + return "" + milvus_expr = getattr(where, "milvus", None) + if isinstance(milvus_expr, str): + return milvus_expr + if not isinstance(where, str): + return str(where) + return _lancedb_expr_to_milvus(where) + + +def _lancedb_expr_to_milvus(expr: str) -> str: + """Best-effort conversion for internal EverOS LanceDB predicates.""" + converted = expr + converted = re.sub(r"\barray_has\s*\(", "array_contains(", converted) + converted = re.sub(r"\bIS\s+NULL\b", "is null", converted, flags=re.IGNORECASE) + converted = re.sub( + r"\b([A-Za-z_][A-Za-z0-9_]*)\s+IN\s+\(([^)]*)\)", + lambda m: f"{m.group(1)} in [{m.group(2)}]", + converted, + flags=re.IGNORECASE, + ) + converted = re.sub(r"(?=])=(?!=)", "==", converted) + converted = re.sub(r"\btimestamp\b", "timestamp_ms", converted) + converted = re.sub(r"\bcreated_at\b", "created_at_ms", converted) + converted = re.sub(r"\bupdated_at\b", "updated_at_ms", converted) + converted = re.sub( + r"TIMESTAMP\s+'([^']*)'", + lambda m: str(_iso_to_ms(m.group(1))), + converted, + ) + return converted + + +def _datetime_storage_field(name: str) -> str: + return f"{name}_ms" + + +def _sparse_field(name: str) -> str: + return f"{name}{_SPARSE_SUFFIX}" + + +def _datetime_to_ms(value: Any) -> int: + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") + + +def _iso_to_ms(value: str) -> int: + aware = ensure_utc(from_iso_format(value)) + assert aware is not None + return to_timestamp_ms(aware) + + +def _is_optional(annotation: Any) -> bool: + args = get_args(annotation) + return type(None) in args + + +def _non_none_args(annotation: Any) -> tuple[Any, ...]: + return tuple(arg for arg in get_args(annotation) if arg is not type(None)) + + +def _is_datetime(annotation: Any) -> bool: + if annotation is dt.datetime: + return True + return any(arg is dt.datetime for arg in _non_none_args(annotation)) + + +def _is_list(annotation: Any) -> bool: + origin = get_origin(annotation) + if origin is list: + return True + return any(get_origin(arg) is list for arg in _non_none_args(annotation)) + + +def _is_float(annotation: Any) -> bool: + if annotation is float: + return True + return any(arg is float for arg in _non_none_args(annotation)) + + +def _is_int(annotation: Any) -> bool: + if annotation is int: + return True + return any(arg is int for arg in _non_none_args(annotation)) + + +def _sort_value(value: Any) -> Any: + if value is None: + fallback = ensure_utc(dt.datetime.min) + assert fallback is not None + return fallback + return value + + +def _bm25_score_from_distance(distance: Any) -> float: + """Convert the Milvus BM25 search distance into a higher-is-better score.""" + if distance is None: + return 0.0 + return abs(float(distance)) + + +def _cosine_distance_from_milvus(distance: Any) -> float | None: + """Normalize Milvus COSINE results to LanceDB-style distance. + + Milvus server / Zilliz Cloud return cosine similarity through the + ``distance`` field. Milvus Lite 3.0 reports cosine distance instead; keep + this version-gated workaround narrow because the upstream issue is expected + to be fixed after 3.0: + https://github.com/milvus-io/milvus-lite/issues/343 + + Recaller code expects LanceDB's ``1 - similarity`` distance contract. + """ + if distance is None: + return None + value = float(distance) + if not _uses_milvus_lite_3_0_cosine_distance_bug(): + value = 1.0 - value + if value < 0.0: + return 0.0 + if value > 1.0: + return 1.0 + return value + + +def _uses_milvus_lite() -> bool: + uri = load_settings().milvus.uri + if not uri: + return True + return "://" not in uri + + +def _uses_milvus_lite_3_0_cosine_distance_bug() -> bool: + if not _uses_milvus_lite(): + return False + return _milvus_lite_version() in {"3.0", "3.0.0"} + + +def _milvus_lite_version() -> str | None: + try: + return version("milvus-lite") + except PackageNotFoundError: + return None + + +def _first_result_set(results: Any) -> list[dict[str, Any]]: + if not results: + return [] + first = results[0] + return list(first or []) + + +async def _run(func: Any, /, *args: Any, **kwargs: Any) -> Any: + return await asyncio.to_thread(func, *args, **kwargs) + + +__all__ = ["MilvusRepoBase"] diff --git a/src/everos/memory/cascade/handlers/agent_case.py b/src/everos/memory/cascade/handlers/agent_case.py index 453c053d5..275e30fa1 100644 --- a/src/everos/memory/cascade/handlers/agent_case.py +++ b/src/everos/memory/cascade/handlers/agent_case.py @@ -26,7 +26,7 @@ from __future__ import annotations -from everos.infra.persistence.lancedb import AgentCase, ParentType, agent_case_repo +from everos.infra.persistence.index import AgentCase, ParentType, agent_case_repo from ._common import require_float, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 12046d526..c8831b2a2 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -38,7 +38,7 @@ import anyio from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import AgentSkill, agent_skill_repo from everos.infra.persistence.markdown import AgentSkillFrontmatter from ..types import HandlerOutcome diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index a33e94ede..4389555c5 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -19,7 +19,7 @@ from __future__ import annotations -from everos.infra.persistence.lancedb import AtomicFact, ParentType, atomic_fact_repo +from everos.infra.persistence.index import AtomicFact, ParentType, atomic_fact_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index e28ac8c87..4750afece 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -32,7 +32,7 @@ import asyncio -from everos.infra.persistence.lancedb import Episode, ParentType, episode_repo +from everos.infra.persistence.index import Episode, ParentType, episode_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index 50bae7728..b9dab218e 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -26,7 +26,7 @@ from __future__ import annotations -from everos.infra.persistence.lancedb import Foresight, ParentType, foresight_repo +from everos.infra.persistence.index import Foresight, ParentType, foresight_repo from ._common import ( optional_int, diff --git a/src/everos/memory/cascade/handlers/knowledge_topic.py b/src/everos/memory/cascade/handlers/knowledge_topic.py index 80e2cdfd4..254124ed8 100644 --- a/src/everos/memory/cascade/handlers/knowledge_topic.py +++ b/src/everos/memory/cascade/handlers/knowledge_topic.py @@ -34,7 +34,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.persistence import MarkdownReader, ParsedMarkdown -from everos.infra.persistence.lancedb import KnowledgeTopic, knowledge_topic_repo +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from everos.infra.persistence.sqlite import ( TopicUpsertPayload, knowledge_topic_sqlite_repo, diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 4842986d0..8d7573521 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -27,7 +27,7 @@ from typing import Any, ClassVar from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import UserProfile, user_profile_repo +from everos.infra.persistence.index import UserProfile, user_profile_repo from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 diff --git a/src/everos/memory/cascade/registry.py b/src/everos/memory/cascade/registry.py index 34de209e3..e74162017 100644 --- a/src/everos/memory/cascade/registry.py +++ b/src/everos/memory/cascade/registry.py @@ -19,7 +19,7 @@ from pathlib import PurePosixPath from everos.core.persistence.markdown import BaseFrontmatter -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 407a06bc7..46b1e6305 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -16,6 +16,7 @@ from __future__ import annotations from everos.memory.search import FilterNode, compile_filters +from everos.memory.search.filters import BackendFilters, compile_filters_for_backends def compile_filters_for_get( @@ -38,3 +39,21 @@ def compile_filters_for_get( app_id=app_id, project_id=project_id, ) + + +def compile_filters_for_get_backends( + filters: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> BackendFilters: + """Compile ``/get`` filters for every derived index backend.""" + return compile_filters_for_backends( + filters, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index 1000cbdd2..e4ddaa7f8 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -12,7 +12,7 @@ Reads only — never writes. Filters are compiled through :func:`compile_filters_for_get` so the column allow-list stays shared with :mod:`memory.search`. Pagination + in-memory sort -runs through :meth:`LanceRepoBase.find_where_paginated`. +runs through the configured derived index repository. """ from __future__ import annotations @@ -34,11 +34,11 @@ GetRequest, GetResponse, ) -from .filters_adapter import compile_filters_for_get +from .filters_adapter import compile_filters_for_get_backends if TYPE_CHECKING: from everos.core.persistence.lancedb import LanceRepoBase - from everos.infra.persistence.lancedb import ( + from everos.infra.persistence.index import ( AgentCase, AgentSkill, Episode, @@ -49,7 +49,7 @@ class GetManager: - """Dispatch ``GetRequest`` to the matching LanceDB-backed repo and + """Dispatch ``GetRequest`` to the matching derived index repo and shape rows into the public DTO.""" def __init__( @@ -70,7 +70,7 @@ def __init__( async def get(self, req: GetRequest) -> GetResponse: request_id = gen_request_id() descending = req.sort_order == "desc" - where = compile_filters_for_get( + where = compile_filters_for_get_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, @@ -187,7 +187,7 @@ def _shape_agent_skill(row: AgentSkill) -> GetAgentSkillItem: async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: """Fetch the owner's single profile row from the ``user_profile`` - LanceDB table (kept in sync with ``users//user.md`` by cascade). + Derived index row (kept in sync with ``users//user.md`` by cascade). Profile is one-row-per-owner KV — there is no pagination / sort / filter surface, so at most one item is returned. Mirrors the diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 2dd9aa804..22df21a47 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,4 +1,4 @@ -"""Filters DSL → LanceDB ``where`` string compiler. +"""Filters DSL → derived-index filter compiler. The Filters DSL is intentionally permissive at the JSON layer (so callers can pass whatever they like and get a clean 400 if it is not supported) @@ -23,7 +23,8 @@ kind). Iterate / membership-test only; do not mutate. * :data:`RESERVED_FIELDS` — names rejected inside any ``filters`` block. * :func:`compile_predicate` — render one ``{field: value}`` clause to - SQL. Operator-map and equality-shorthand are both handled. + a backend-specific predicate. Operator-map and equality-shorthand are + both handled. The high-level :func:`compile_filters` remains the entry point for ``/search`` (combinator-aware). @@ -31,10 +32,18 @@ from __future__ import annotations +import dataclasses import datetime as _dt +import json from typing import Any, Final -from everos.component.utils.datetime import from_timestamp, to_iso_format +from everos.component.utils.datetime import ( + ensure_utc, + from_iso_format, + from_timestamp, + to_iso_format, + to_timestamp_ms, +) from everos.core.errors import FilterError as FilterError from .dto import FilterNode @@ -50,6 +59,15 @@ "lte": "<=", "in": "IN", } +_MILVUS_OP_MAP: Final[dict[str, str]] = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", + "in": "in", +} # Field kinds: ``str`` rendered as ``''``; ``ts`` rendered as # ``TIMESTAMP ''`` (DataFusion timestamp literal); ``array_str`` @@ -82,6 +100,17 @@ def __init__(self, column: str, kind: _FieldKind) -> None: ) +@dataclasses.dataclass(frozen=True) +class BackendFilters: + """Compiled filters for every supported derived index backend.""" + + lancedb: str + milvus: str + + def __str__(self) -> str: + return self.lancedb + + # ── Public API ─────────────────────────────────────────────────────────── @@ -102,19 +131,72 @@ def compile_filters( from another — omitting it would let a query bleed across spaces. Both ``/search`` and ``/get`` share this compile path. """ - base = [ - f"owner_id = '{_escape_str(owner_id)}'", - f"owner_type = '{owner_type}'", - f"app_id = '{_escape_str(app_id)}'", - f"project_id = '{_escape_str(project_id)}'", - ] + return _compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="lancedb", + ) + + +def compile_filters_for_backends( + node: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> BackendFilters: + """Compile request filters for every supported derived index backend.""" + return BackendFilters( + lancedb=_compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="lancedb", + ), + milvus=_compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="milvus", + ), + ) + + +def _compile_filters_backend( + node: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + backend: str, +) -> str: + base = _base_clauses( + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend=backend, + ) # Only episode / atomic_fact tables carry the ``deprecated_by`` column # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": - base.append("deprecated_by IS NULL") + base.append( + "deprecated_by IS NULL" + if backend == "lancedb" + else "deprecated_by is null" + ) if node is None: return " AND ".join(base) - compiled = _compile_node(node.model_dump(exclude_none=True)) + compiled = _compile_node(node.model_dump(exclude_none=True), backend=backend) if not compiled: return " AND ".join(base) return " AND ".join([*base, compiled]) @@ -123,7 +205,7 @@ def compile_filters( # ── Internals ──────────────────────────────────────────────────────────── -def _compile_node(raw: dict[str, Any]) -> str: +def _compile_node(raw: dict[str, Any], *, backend: str = "lancedb") -> str: """Walk one DSL node; return the matching SQL fragment (no leading parens). Empty nodes yield ``""`` so :func:`compile_filters` can skip the @@ -133,9 +215,9 @@ def _compile_node(raw: dict[str, Any]) -> str: parts: list[str] = [] if (and_list := raw.pop("AND", None)) is not None: - parts.append(_compile_combinator(and_list, "AND")) + parts.append(_compile_combinator(and_list, "AND", backend=backend)) if (or_list := raw.pop("OR", None)) is not None: - parts.append(_compile_combinator(or_list, "OR")) + parts.append(_compile_combinator(or_list, "OR", backend=backend)) for field, value in raw.items(): if field in RESERVED_FIELDS: @@ -144,7 +226,7 @@ def _compile_node(raw: dict[str, Any]) -> str: ) if field not in ALLOWED_FIELDS: raise FilterError(f"unsupported filter field: {field!r}") - parts.append(compile_predicate(field, value)) + parts.append(compile_predicate(field, value, backend=backend)) # Drop empty fragments coming from empty AND/OR arrays. parts = [p for p in parts if p] @@ -155,7 +237,9 @@ def _compile_node(raw: dict[str, Any]) -> str: return " AND ".join(parts) -def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: +def _compile_combinator( + children: list[dict[str, Any]], op: str, *, backend: str = "lancedb" +) -> str: """Render an ``AND`` / ``OR`` array of child nodes.""" if not isinstance(children, list): raise FilterError(f"{op} expects an array of nodes") @@ -163,7 +247,7 @@ def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: for child in children: if not isinstance(child, dict): raise FilterError(f"{op} children must be objects") - compiled = _compile_node(child) + compiled = _compile_node(child, backend=backend) if compiled: fragments.append(f"({compiled})") if not fragments: @@ -175,7 +259,7 @@ def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: return "(" + glue.join(fragments) + ")" -def compile_predicate(field: str, value: Any) -> str: +def compile_predicate(field: str, value: Any, *, backend: str = "lancedb") -> str: """Render one ``"": `` clause to SQL. Public primitive — :mod:`memory.get` builds a flat (no AND/OR) @@ -195,48 +279,65 @@ def compile_predicate(field: str, value: Any) -> str: if not value: raise FilterError(f"empty operator map for field {field!r}") clauses = [ - _compile_op_clause(spec, field, op, op_val) for op, op_val in value.items() + _compile_op_clause(spec, field, op, op_val, backend=backend) + for op, op_val in value.items() ] if len(clauses) == 1: return clauses[0] return "(" + " AND ".join(clauses) + ")" # Equality shorthand. - return _compile_op_clause(spec, field, "eq", value) + return _compile_op_clause(spec, field, "eq", value, backend=backend) -def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> str: +def _compile_op_clause( + spec: _FieldSpec, field: str, op: str, value: Any, *, backend: str = "lancedb" +) -> str: """Render a single `` `` clause.""" if op not in _OP_MAP: raise FilterError(f"unsupported operator {op!r} on field {field!r}") - sql_op = _OP_MAP[op] + op_map = _MILVUS_OP_MAP if backend == "milvus" else _OP_MAP + sql_op = op_map[op] + column = _column_for_backend(spec, backend) if spec.kind == "array_str": # Only equality / membership make sense on a list column. + fn = "array_contains" if backend == "milvus" else "array_has" if op == "eq": - literal = _escape_str(_require_str(value, field)) - return f"array_has({spec.column}, '{literal}')" + literal = _render_str_literal(_require_str(value, field), backend) + return f"{fn}({column}, {literal})" if op == "in": items = _require_list(value, field) - literals = [f"'{_escape_str(_require_str(v, field))}'" for v in items] - inner = " OR ".join(f"array_has({spec.column}, {lit})" for lit in literals) + literals = [ + _render_str_literal(_require_str(v, field), backend) for v in items + ] + inner = " OR ".join(f"{fn}({column}, {lit})" for lit in literals) return f"({inner})" raise FilterError(f"operator {op!r} is not supported on array field {field!r}") if op == "in": items = _require_list(value, field) - literals = [_render_literal(v, spec.kind, field) for v in items] - return f"{spec.column} IN ({', '.join(literals)})" + literals = [ + _render_literal(v, spec.kind, field, backend=backend) for v in items + ] + if backend == "milvus": + return f"{column} in [{', '.join(literals)}]" + return f"{column} IN ({', '.join(literals)})" - return f"{spec.column} {sql_op} {_render_literal(value, spec.kind, field)}" + literal = _render_literal(value, spec.kind, field, backend=backend) + return f"{column} {sql_op} {literal}" # ── Literal rendering ──────────────────────────────────────────────────── -def _render_literal(value: Any, kind: _FieldKind, field: str) -> str: +def _render_literal( + value: Any, kind: _FieldKind, field: str, *, backend: str = "lancedb" +) -> str: if kind == "str": - return f"'{_escape_str(_require_str(value, field))}'" + return _render_str_literal(_require_str(value, field), backend) if kind == "ts": + if backend == "milvus": + return str(_render_ts_ms(value, field)) return f"TIMESTAMP '{_render_ts(value, field)}'" raise FilterError(f"unsupported field kind {kind!r} for field {field!r}") @@ -257,6 +358,58 @@ def _render_ts(value: Any, field: str) -> str: raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") +def _render_ts_ms(value: Any, field: str) -> int: + if isinstance(value, bool): + raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, str): + if "'" in value: + raise FilterError(f"timestamp string for {field!r} contains a quote") + try: + aware = ensure_utc(from_iso_format(value)) + except (TypeError, ValueError) as exc: + raise FilterError( + f"timestamp value for {field!r} must be ms or ISO string" + ) from exc + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, _dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") + + +def _render_str_literal(value: str, backend: str) -> str: + if backend == "milvus": + return json.dumps(value, ensure_ascii=False) + return f"'{_escape_str(value)}'" + + +def _base_clauses( + *, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + backend: str, +) -> list[str]: + eq = "==" if backend == "milvus" else "=" + return [ + f"owner_id {eq} {_render_str_literal(owner_id, backend)}", + f"owner_type {eq} {_render_str_literal(owner_type, backend)}", + f"app_id {eq} {_render_str_literal(app_id, backend)}", + f"project_id {eq} {_render_str_literal(project_id, backend)}", + ] + + +def _column_for_backend(spec: _FieldSpec, backend: str) -> str: + if backend == "milvus" and spec.kind == "ts": + return f"{spec.column}_ms" + return spec.column + + def _escape_str(value: str) -> str: """Double single quotes — SQL-standard escape for a single-quoted literal.""" return value.replace("'", "''") diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 7633df415..e86672c7b 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -22,7 +22,8 @@ no query-relevance score we can assign to a fact pulled by parent_id alone, and emitting ``score=0.0`` facts would muddy the contract. -The manager never writes to storage; it only reads LanceDB + markdown. +The manager never writes to storage; it only reads the derived index + +markdown. """ from __future__ import annotations @@ -59,7 +60,7 @@ SearchResponse, UnprocessedMessageDTO, ) -from .filters import compile_filters +from .filters import compile_filters_for_backends from .hierarchy import build_ep_to_fact_parents, heap_expand from .shaper import ( reshape_hybrid_output, @@ -156,7 +157,7 @@ async def search(self, req: SearchRequest) -> SearchResponse: # Compile filters first: a malformed `filters` payload is a user # input error (422) and should surface before the server-side # component guard (500). The two steps are independent. - where = compile_filters( + where = compile_filters_for_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, diff --git a/src/everos/memory/search/recall/agent_case.py b/src/everos/memory/search/recall/agent_case.py index 93a7f9b60..9c17ba45e 100644 --- a/src/everos/memory/search/recall/agent_case.py +++ b/src/everos/memory/search/recall/agent_case.py @@ -13,17 +13,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentCase, get_table +from everos.infra.persistence.index import AgentCase, agent_case_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -50,43 +48,15 @@ async def sparse_recall( per BM25 column (``MatchQuery`` is column-bound), then the two per-column result lists merge by id keeping the max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentCase.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentCase.BM25_FIELDS), + merged_rows = await agent_case_repo.sparse_search( + terms, + where, + columns=AgentCase.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep the max BM25 score across the two columns. - # task_intent hits typically score higher (the retrieval anchor); - # approach hits catch queries that match a step detail. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -97,15 +67,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_case_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index f444e4a38..0c1831c79 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -9,17 +9,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentSkill, get_table +from everos.infra.persistence.index import AgentSkill, agent_skill_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -48,41 +46,15 @@ async def sparse_recall( rationale. One BooleanQuery per BM25 column; merge by id with max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentSkill.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentSkill.BM25_FIELDS), + merged_rows = await agent_skill_repo.sparse_search( + terms, + where, + columns=AgentSkill.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep max BM25 score across the two columns. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -93,15 +65,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_skill_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -124,8 +88,7 @@ async def fetch_by_case_ids( """ if not case_ids: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) clause = " OR ".join(f"array_has(source_case_ids, '{_q(c)}')" for c in case_ids) full_where = f"({where}) AND ({clause})" - rows = await table.query().where(full_where).limit(limit).to_list() + rows = await agent_skill_repo.search(where=full_where, limit=limit) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 69eba22d4..38ec3872a 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -21,11 +21,10 @@ from everalgo.types import Candidate, FactCandidate -from everos.infra.persistence.lancedb import AtomicFact, get_table +from everos.infra.persistence.index import AtomicFact, atomic_fact_repo from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -49,14 +48,14 @@ async def sparse_recall( self, query: str, where: str, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery (see EpisodeRecaller docstring).""" - bq = build_or_query( - self._deps.tokenizer, query, column=AtomicFact.BM25_FIELDS[0] - ) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await atomic_fact_repo.sparse_search( + terms, + where, + columns=AtomicFact.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -78,15 +77,7 @@ async def dense_recall( """ if not vector: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await atomic_fact_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -170,17 +161,11 @@ async def _query_facts_for_parents( clause = f"parent_id IN ({quoted})" full_where = f"({where}) AND ({clause})" limit = per_episode * max(len(parent_to_eps), 1) - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) if query_vector: - return await ( - table.query() - .nearest_to(list(query_vector)) - .distance_type("cosine") - .where(full_where) - .limit(limit) - .to_list() + return await atomic_fact_repo.dense_search( + query_vector, full_where, limit=limit ) - return await table.query().where(full_where).limit(limit).to_list() + return await atomic_fact_repo.search(where=full_where, limit=limit) def _build_parent_to_episode_map( diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index a2766dcec..1f2c13821 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,11 +7,10 @@ from everalgo.types import Candidate -from everos.infra.persistence.lancedb import Episode, get_table +from everos.infra.persistence.index import Episode, active_backend, episode_repo from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -55,12 +54,14 @@ async def sparse_recall( Mirrors enterprise's ``bool.should + minimum_should_match=1`` ES design. """ - bq = build_or_query(self._deps.tokenizer, query, column=Episode.BM25_FIELDS[0]) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await episode_repo.sparse_search( + terms, + where, + columns=Episode.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -72,16 +73,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await episode_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -113,6 +105,10 @@ async def dense_recall_subject( """ if not vector: return [] + if active_backend() == "milvus": + return await self.dense_recall(vector, where, limit=limit) + from everos.infra.persistence.lancedb import get_table + table = await get_table(Episode.TABLE_NAME, Episode) rows = ( await table.query() @@ -149,8 +145,7 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: No ``limit`` — the full owner partition is required for cluster membership matching. """ - table = await get_table(Episode.TABLE_NAME, Episode) - rows = await table.query().where(where).to_list() + rows = await episode_repo.search(where=where, limit=10_000) result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") @@ -173,8 +168,7 @@ async def fetch_by_entry_ids( """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" if not entry_ids: return [] - table = await get_table(Episode.TABLE_NAME, Episode) quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) full_where = f"({where}) AND (entry_id IN ({quoted}))" - rows = await table.query().where(full_where).limit(len(entry_ids)).to_list() + rows = await episode_repo.search(where=full_where, limit=len(entry_ids)) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py index ea484e17a..9cc6a9bfb 100644 --- a/src/everos/memory/search/recall/knowledge_topic.py +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -12,17 +12,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import KnowledgeTopic, get_table +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -78,26 +76,15 @@ async def sparse_recall( matching the query in either its summary or its content body is surfaced without double-counting. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, KnowledgeTopic.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in KnowledgeTopic.BM25_FIELDS), + merged_rows = await knowledge_topic_repo.sparse_search( + terms, + where, + columns=KnowledgeTopic.BM25_FIELDS, + limit=limit, ) - merged_rows = _merge_bm25_results(per_column, limit=limit) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -109,15 +96,7 @@ async def dense_recall( """Cosine ANN over the ``summary`` vector (1024-d).""" if not vector: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await knowledge_topic_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/profile.py b/src/everos/memory/search/recall/profile.py index be149b927..640d27091 100644 --- a/src/everos/memory/search/recall/profile.py +++ b/src/everos/memory/search/recall/profile.py @@ -19,7 +19,7 @@ from typing import Any from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import user_profile_repo +from everos.infra.persistence.index import user_profile_repo from ..dto import SearchProfileItem diff --git a/src/everos/memory/strategies/extract_agent_skill.py b/src/everos/memory/strategies/extract_agent_skill.py index ceccf5dfb..b9d5cb164 100644 --- a/src/everos/memory/strategies/extract_agent_skill.py +++ b/src/everos/memory/strategies/extract_agent_skill.py @@ -49,13 +49,13 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase as LanceAgentCase, ) -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentSkill as LanceAgentSkill, ) -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, ) @@ -205,7 +205,7 @@ async def _load_target_case( if target is None: # Cascade hasn't indexed the freshly-written md yet. raise _CaseNotYetIndexedError( - f"AgentCase entry_id={case_entry_id} not in LanceDB yet; retrying" + f"AgentCase entry_id={case_entry_id} not in derived index yet; retrying" ) return target diff --git a/src/everos/memory/strategies/extract_user_profile.py b/src/everos/memory/strategies/extract_user_profile.py index c4c9a6e29..7cc064548 100644 --- a/src/everos/memory/strategies/extract_user_profile.py +++ b/src/everos/memory/strategies/extract_user_profile.py @@ -37,7 +37,7 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import episode_repo +from everos.infra.persistence.index import episode_repo from everos.infra.persistence.markdown import ( ProfileReader, ProfileWriter, diff --git a/src/everos/memory/strategies/reflect_episodes.py b/src/everos/memory/strategies/reflect_episodes.py index 8f72f2ac3..5a208e822 100644 --- a/src/everos/memory/strategies/reflect_episodes.py +++ b/src/everos/memory/strategies/reflect_episodes.py @@ -22,7 +22,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import CronTick from everos.infra.ome.triggers import Cron -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( atomic_fact_repo, episode_repo, ) diff --git a/src/everos/service/get.py b/src/everos/service/get.py index 9b9a7e679..3265f9eaa 100644 --- a/src/everos/service/get.py +++ b/src/everos/service/get.py @@ -1,8 +1,8 @@ """Get use case — lazy singleton wiring for ``POST /api/v1/memory/get``. Mirrors :mod:`everos.service.search`: the :class:`GetManager` and its -LanceDB repo singletons are built on first call so the FastAPI module -import order stays decoupled from the lifespan that brings up LanceDB. +derived index repo singletons are built on first call so the FastAPI module +import order stays decoupled from the lifespan that brings up the index backend. ``/get`` is read-only and uses no embedding / LLM / rerank clients — it never blocks on optional components the way ``/search`` does. @@ -11,7 +11,7 @@ from __future__ import annotations from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, episode_repo, diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index fb4fbb17f..b02d2e78b 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -120,6 +120,33 @@ def test_embedding_rerank_defaults() -> None: assert s.llm.api_key.get_secret_value() == "" +def test_index_milvus_defaults() -> None: + s = Settings() + assert s.index.backend == "lancedb" + assert s.milvus.uri == "" + assert s.milvus.token.get_secret_value() == "" + assert s.milvus.db_name == "" + assert s.milvus.consistency_level == "Session" + assert s.milvus.dimension == 1024 + assert s.milvus.collection_prefix == "everos" + + +def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://localhost:19530") + monkeypatch.setenv("EVEROS_MILVUS__TOKEN", "secret") + monkeypatch.setenv("EVEROS_MILVUS__DB_NAME", "tenant_a") + monkeypatch.setenv("EVEROS_MILVUS__CONSISTENCY_LEVEL", "Strong") + monkeypatch.setenv("EVEROS_MILVUS__DIMENSION", "1536") + s = Settings() + assert s.index.backend == "milvus" + assert s.milvus.uri == "http://localhost:19530" + assert s.milvus.token.get_secret_value() == "secret" + assert s.milvus.db_name == "tenant_a" + assert s.milvus.consistency_level == "Strong" + assert s.milvus.dimension == 1536 + + def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: """No --root, no EVEROS_ROOT → ~/.everos.""" monkeypatch.delenv("EVEROS_ROOT", raising=False) diff --git a/tests/unit/test_core/test_persistence/test_memory_root.py b/tests/unit/test_core/test_persistence/test_memory_root.py index 8dd6dad15..cf36f24a3 100644 --- a/tests/unit/test_core/test_persistence/test_memory_root.py +++ b/tests/unit/test_core/test_persistence/test_memory_root.py @@ -67,6 +67,8 @@ def test_dotfile_paths(tmp_path: Path) -> None: mr = MemoryRoot(tmp_path) assert mr.index_dir == tmp_path / ".index" assert mr.lancedb_dir == tmp_path / ".index" / "lancedb" + assert mr.milvus_dir == tmp_path / ".index" / "milvus" + assert mr.milvus_db == tmp_path / ".index" / "milvus" / "milvus.db" assert mr.sqlite_dir == tmp_path / ".index" / "sqlite" assert mr.system_db == tmp_path / ".index" / "sqlite" / "system.db" assert mr.lock_file == tmp_path / ".lock" @@ -80,6 +82,7 @@ def test_ensure_creates_required_dirs(tmp_path: Path) -> None: assert mr.index_dir.is_dir() assert mr.sqlite_dir.is_dir() assert mr.lancedb_dir.is_dir() + assert mr.milvus_dir.is_dir() assert mr.tmp_dir.is_dir() # User-visible dirs are NOT pre-created. assert not mr.agents_dir().exists() diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py new file mode 100644 index 000000000..901d67034 --- /dev/null +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -0,0 +1,127 @@ +"""Real Milvus Lite coverage for the optional derived index backend.""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +import uuid +from pathlib import Path + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("pymilvus") is None, + reason="pymilvus optional extra is not installed", +) + + +def test_milvus_score_normalization_handles_lite_and_cloud( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from everos.infra.persistence.milvus import repository + + monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(-1.5) == pytest.approx(1.5) + + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.0") + assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) + + monkeypatch.setenv("EVEROS_MILVUS__URI", "https://example.zillizcloud.com") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) + + monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.1") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + + +@pytest_asyncio.fixture(autouse=True) +async def _milvus_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", f"test_{uuid.uuid4().hex}") + load_settings.cache_clear() + yield + from everos.infra.persistence.index import shutdown + + await shutdown() + load_settings.cache_clear() + + +async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: + from everos.infra.persistence.index import Episode, episode_repo + from everos.memory.search.filters import compile_filters_for_backends + + v_apple = [0.0] * 1024 + v_apple[0] = 1.0 + v_banana = [0.0] * 1024 + v_banana[1] = 1.0 + now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + + await episode_repo.upsert( + [ + Episode( + id="u1_ep1", + entry_id="ep1", + owner_id="u1", + owner_type="user", + app_id="default", + project_id="default", + session_id="s1", + timestamp=now, + parent_id="mc1", + sender_ids=["user"], + episode="red apple memory", + episode_tokens="red apple memory", + md_path="default_app/default_project/users/u1/episodes/day.md", + content_sha256="a", + vector=v_apple, + ), + Episode( + id="u1_ep2", + entry_id="ep2", + owner_id="u1", + owner_type="user", + app_id="default", + project_id="default", + session_id="s1", + timestamp=now + dt.timedelta(seconds=1), + parent_id="mc2", + sender_ids=["assistant"], + episode="blue banana memory", + episode_tokens="blue banana memory", + md_path="default_app/default_project/users/u1/episodes/day.md", + content_sha256="b", + vector=v_banana, + ), + ] + ) + + filters = compile_filters_for_backends(None, owner_id="u1", owner_type="user") + rows = await episode_repo.find_where(filters, limit=10) + assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} + assert rows[0].timestamp.utcoffset() == dt.timedelta(0) + + sparse = await episode_repo.sparse_search( + ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 + ) + assert [row["id"] for row in sparse] == ["u1_ep1"] + assert sparse[0]["_score"] > 0 + + dense = await episode_repo.dense_search(v_apple, filters, limit=5) + assert dense[0]["id"] == "u1_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0) + + deleted = await episode_repo.delete_by_md_path( + "default_app/default_project/users/u1/episodes/day.md" + ) + assert deleted == 2 + assert await episode_repo.count() == 0 diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index cb4e16714..93830a688 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -56,7 +56,7 @@ class _StubRepo: async def find_where_paginated( self, - where: str, + where: Any, *, sort_by: str, descending: bool = True, @@ -65,7 +65,7 @@ async def find_where_paginated( max_fetch: int = 20000, ) -> tuple[list[Any], int]: self.last = _CallRecord( - where=where, + where=str(where), sort_by=sort_by, descending=descending, page=page, diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 808c2a382..a4267d8c8 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -9,6 +9,7 @@ FilterNode, compile_filters, ) +from everos.memory.search.filters import compile_filters_for_backends # ── Base injection ─────────────────────────────────────────────────────── @@ -266,3 +267,35 @@ def test_compile_filters_excludes_deprecated_by_for_user() -> None: def test_compile_filters_omits_deprecated_by_for_agent() -> None: result = compile_filters(None, owner_id="agent_1", owner_type="agent") assert "deprecated_by" not in result + + +# ── Backend-specific rendering ───────────────────────────────────────── + + +def test_compile_filters_for_backends_preserves_lancedb_default() -> None: + filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") + assert str(filters) == filters.lancedb + assert "owner_id = 'u_a'" in filters.lancedb + assert "owner_id == \"u_a\"" in filters.milvus + assert "deprecated_by IS NULL" in filters.lancedb + assert "deprecated_by is null" in filters.milvus + + +def test_compile_filters_for_milvus_timestamp_and_array() -> None: + node = FilterNode.model_validate( + {"timestamp": {"gte": 1704067200000}, "sender_id": "u_jason"} + ) + filters = compile_filters_for_backends(node, owner_id="u_a", owner_type="user") + assert "timestamp_ms >= 1704067200000" in filters.milvus + assert 'array_contains(sender_ids, "u_jason")' in filters.milvus + assert "TIMESTAMP '" in filters.lancedb + assert "array_has(sender_ids, 'u_jason')" in filters.lancedb + + +def test_compile_filters_for_milvus_escapes_string_literals() -> None: + node = FilterNode.model_validate({"session_id": "ses's"}) + filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") + assert "owner_id == \"al'ice\"" in filters.milvus + assert "session_id == \"ses's\"" in filters.milvus + assert "owner_id = 'al''ice'" in filters.lancedb + assert "session_id = 'ses''s'" in filters.lancedb diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index ff8817be3..7e1d08fdc 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -101,25 +101,25 @@ def __init__(self, sparse: list[Candidate], dense: list[Candidate]) -> None: self.last_where: str | None = None async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = str(where) return list(self._sparse[:limit]) async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = str(where) return list(self._dense[:limit]) async def fetch_by_parent_ids( - self, parent_ids: Sequence[str], where: str + self, parent_ids: Sequence[str], where: Any ) -> list[Candidate]: by_parent = {str(c.metadata.get("parent_id", "")): c for c in self._dense} return [by_parent[p] for p in parent_ids if p in by_parent] async def fetch_by_entry_ids( - self, entry_ids: Sequence[str], where: str + self, entry_ids: Sequence[str], where: Any ) -> list[Candidate]: by_entry = {str(c.metadata.get("entry_id", "")): c for c in self._dense} return [by_entry[e] for e in entry_ids if e in by_entry] @@ -147,7 +147,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Any, *, per_episode: int, query_vector: Any = None, @@ -198,7 +198,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: return list(self._dense) async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Any, *, limit: int ) -> list[Candidate]: return list(self._by_case) diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 4549322d8..2cde9f368 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -58,9 +58,9 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( _make_row("ep_2", "mc_2"), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -75,9 +75,9 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( """metadata['episode_id'] carries the real LanceDB episode id for final shaping.""" rows = [_make_row("ep_abc", "mc_xyz")] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -104,9 +104,9 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( }, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -131,9 +131,9 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -156,9 +156,9 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -179,14 +179,10 @@ async def test_fetch_by_entry_ids_returns_candidates( entry_id="entry_xyz", ), ] - mock_tbl = MagicMock() - mock_tbl.query.return_value.where.return_value.limit.return_value.to_list = ( - AsyncMock(return_value=rows) - ) with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=mock_tbl, + return_value=rows, ): result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") @@ -226,9 +222,9 @@ async def test_sparse_recall_as_child_injects_parent_id( {**_make_row("ep_1", "mc_1", entry_id="entry_1"), "_score": 1.0}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table(rows), + return_value=rows, ): result = await recaller.sparse_recall_as_child( "hello", "owner_id = 'alice'", limit=10 @@ -256,9 +252,9 @@ async def test_sparse_recall_as_child_falls_back_to_id_when_no_entry_id( "_score": 0.5, } with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table([row]), + return_value=[row], ): result = await recaller.sparse_recall_as_child( "hello", "owner_id = 'alice'", limit=10 @@ -286,9 +282,9 @@ async def test_dense_recall_as_child_injects_parent_id( {**_make_row("ep_3", "mc_3", entry_id="entry_3"), "_distance": 0.1}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_ann_table(rows), + return_value=rows, ): result = await recaller.dense_recall_as_child( [0.1] * 1024, "owner_id = 'alice'", limit=10 @@ -325,10 +321,16 @@ async def test_dense_recall_subject_returns_subject_vector_source( rows = [ {**_make_row("ep_s1", "mc_s1", entry_id="entry_s1"), "_distance": 0.2}, ] - with patch( - "everos.memory.search.recall.episode.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + with ( + patch( + "everos.memory.search.recall.episode.active_backend", + return_value="lancedb", + ), + patch( + "everos.infra.persistence.lancedb.get_table", + new_callable=AsyncMock, + return_value=_mock_subject_ann_table(rows), + ), ): result = await recaller.dense_recall_subject( [0.1] * 1024, "owner_id = 'alice'", limit=10 @@ -354,10 +356,16 @@ async def test_dense_recall_subject_as_child_injects_parent_id( rows = [ {**_make_row("ep_s2", "mc_s2", entry_id="entry_s2"), "_distance": 0.15}, ] - with patch( - "everos.memory.search.recall.episode.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + with ( + patch( + "everos.memory.search.recall.episode.active_backend", + return_value="lancedb", + ), + patch( + "everos.infra.persistence.lancedb.get_table", + new_callable=AsyncMock, + return_value=_mock_subject_ann_table(rows), + ), ): result = await recaller.dense_recall_subject_as_child( [0.1] * 1024, "owner_id = 'alice'", limit=10 diff --git a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py index d40d8a41e..3e98c58b4 100644 --- a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py +++ b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py @@ -1,11 +1,11 @@ """Unit tests for ``KnowledgeTopicRecaller``. -Verifies dual-column BM25 + cosine ANN recall, using ``unittest.mock`` -to patch ``get_table`` so no real LanceDB connection is needed. +Verifies dual-column BM25 delegation + cosine ANN recall, using +``unittest.mock`` to patch the backend-neutral index repo. White-box surfaces touched: - - ``everos.memory.search.recall.knowledge_topic.get_table`` (patched) - - ``KnowledgeTopicRecaller.sparse_recall`` — queries both BM25 columns + - ``everos.memory.search.recall.knowledge_topic.knowledge_topic_repo`` (patched) + - ``KnowledgeTopicRecaller.sparse_recall`` — passes both BM25 columns - ``KnowledgeTopicRecaller.dense_recall`` — cosine ANN with distance→score """ @@ -17,6 +17,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index import KnowledgeTopic from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.knowledge_topic import KnowledgeTopicRecaller @@ -58,41 +59,6 @@ def _make_row( return row -def _mock_bm25_table( - summary_rows: list[dict[str, Any]], - content_rows: list[dict[str, Any]], -) -> MagicMock: - """Build a table mock whose BM25 results differ per column. - - The first ``nearest_to_text`` call (summary_tokens) returns - ``summary_rows``; the second (content_tokens) returns ``content_rows``. - ``asyncio.gather`` fires both concurrently, so we use ``side_effect`` - on the chain rather than recording call order. - """ - summary_chain = MagicMock() - summary_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=summary_rows - ) - - content_chain = MagicMock() - content_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=content_rows - ) - - tbl = MagicMock() - tbl.query.return_value.nearest_to_text.side_effect = [summary_chain, content_chain] - return tbl - - -def _mock_ann_table(rows: list[dict[str, Any]]) -> MagicMock: - """Build a table mock for ANN (dense) queries.""" - tbl = MagicMock() - ann = tbl.query.return_value.nearest_to.return_value - chain = ann.distance_type.return_value.where.return_value.limit.return_value - chain.to_list = AsyncMock(return_value=rows) - return tbl - - @pytest.fixture() def recaller() -> KnowledgeTopicRecaller: return KnowledgeTopicRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) @@ -109,16 +75,20 @@ def recaller() -> KnowledgeTopicRecaller: async def test_sparse_recall_queries_both_columns( recaller: KnowledgeTopicRecaller, ) -> None: - """``nearest_to_text`` must be called once per BM25 column.""" - tbl = _mock_bm25_table( - summary_rows=[_make_row("t1", score=0.9)], - content_rows=[_make_row("t2", score=0.7)], - ) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + """Both BM25 columns must be delegated to the index repo.""" + rows = [_make_row("t1", score=0.9), _make_row("t2", score=0.7)] + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ) as mock_sparse: result = await recaller.sparse_recall("topic query", _WHERE, limit=10) - # nearest_to_text called twice (once per column) - assert tbl.query.return_value.nearest_to_text.call_count == 2 + mock_sparse.assert_awaited_once() + assert list(mock_sparse.await_args.args[0]) == ["topic", "query"] + assert mock_sparse.await_args.args[1] == _WHERE + assert mock_sparse.await_args.kwargs["columns"] == KnowledgeTopic.BM25_FIELDS + assert mock_sparse.await_args.kwargs["limit"] == 10 ids = {c.id for c in result} assert ids == {"t1", "t2"} @@ -126,13 +96,14 @@ async def test_sparse_recall_queries_both_columns( async def test_sparse_recall_merges_by_max_score( recaller: KnowledgeTopicRecaller, ) -> None: - """When the same id appears in both columns, keep the higher score.""" + """Scores returned by the repo are preserved on keyword candidates.""" shared_id = "topic_shared" - summary_rows = [_make_row(shared_id, score=0.5)] - content_rows = [_make_row(shared_id, score=0.9)] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=[_make_row(shared_id, score=0.9)], + ): result = await recaller.sparse_recall("overlap", _WHERE, limit=10) assert len(result) == 1 @@ -144,16 +115,16 @@ async def test_sparse_recall_merges_by_max_score( async def test_sparse_recall_returns_sorted_by_score( recaller: KnowledgeTopicRecaller, ) -> None: - """Merged results must be sorted descending by score, truncated to limit.""" - summary_rows = [ - _make_row("a", score=0.3), + """The recaller preserves repo ordering and maps rows to candidates.""" + rows = [ _make_row("b", score=0.8), - ] - content_rows = [ _make_row("c", score=0.6), ] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.sparse_recall("query", _WHERE, limit=2) assert len(result) == 2 @@ -164,16 +135,19 @@ async def test_sparse_recall_returns_sorted_by_score( async def test_sparse_recall_empty_query_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty tokenisation short-circuits — no LanceDB query is issued.""" + """Empty tokenisation short-circuits — no repo query is issued.""" tok = MagicMock(spec=Tokenizer) tok.tokenize.return_value = [] r = KnowledgeTopicRecaller(RecallerDeps(tokenizer=tok)) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + ) as mock_sparse: result = await r.sparse_recall("", _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_sparse.assert_not_called() # --------------------------------------------------------------------------- @@ -189,8 +163,11 @@ async def test_dense_recall_cosine_conversion( _make_row("t1", distance=0.2), _make_row("t2", distance=0.5), ] - tbl = _mock_ann_table(rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=10) assert len(result) == 2 @@ -203,12 +180,15 @@ async def test_dense_recall_cosine_conversion( async def test_dense_recall_empty_vector_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty vector short-circuits — no LanceDB query is issued.""" - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + """Empty vector short-circuits — no repo query is issued.""" + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + ) as mock_dense: result = await recaller.dense_recall([], _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_dense.assert_not_called() async def test_dense_recall_metadata_excludes_noise_columns( @@ -218,8 +198,11 @@ async def test_dense_recall_metadata_excludes_noise_columns( row = _make_row("t1", distance=0.3) row["vector"] = [0.0] * 1024 - tbl = _mock_ann_table([row]) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=[row], + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=5) assert len(result) == 1 diff --git a/uv.lock b/uv.lock index 65fa5d51f..10f4743e1 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,14 @@ version = 1 revision = 3 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] [[package]] name = "aiosqlite" @@ -99,6 +107,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8b/0d3945a13955303b81272f759a0331e54c5c793da455e6f5706b89d2639c/cachetools-7.1.4.tar.gz", hash = "sha256:437f55a4e0c1b01a4f3077cc470e6991d47430970e36fbcb77e2be0df4fc1cd6", size = 40085, upload-time = "2026-05-21T22:40:43.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/7b/1fc1c09cc0756cf25861a3be10565915953876da48bb228fb9a672b20a42/cachetools-7.1.4-py3-none-any.whl", hash = "sha256:323dc4127934744db5b54eb4924482d7edafbf9554e820d1531c2e08c0e4ef54", size = 16761, upload-time = "2026-05-21T22:40:41.845Z" }, +] + [[package]] name = "cairocffi" version = "1.7.1" @@ -592,6 +609,9 @@ dependencies = [ ] [package.optional-dependencies] +milvus = [ + { name = "pymilvus", extra = ["milvus-lite"] }, +] multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, ] @@ -629,6 +649,7 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, + { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=3.0.0" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -639,7 +660,7 @@ requires-dist = [ { name = "watchdog", specifier = ">=4.0.0" }, { name = "watchfiles", specifier = ">=0.21.0" }, ] -provides-extras = ["multimodal"] +provides-extras = ["multimodal", "milvus"] [package.metadata.requires-dev] dev = [ @@ -663,6 +684,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "faiss-cpu" +version = "1.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, + { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" }, +] + [[package]] name = "fastapi" version = "0.139.0" @@ -824,6 +862,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/8f/774ce522de6a7e70fbeceeaeb6fbe502f5dfb8365728fb3bb4cb23463da8/grimp-3.14-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a424ad14d5deb56721ac24ab939747f72ab3d378d42e7d1f038317d33b052b77", size = 2515157, upload-time = "2025-12-10T17:54:55.874Z" }, ] +[[package]] +name = "grpcio" +version = "1.82.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/bc/656b89387d6f4ed7e0686c7b64c2ae7e554a759aa58122c8e5fb99392c32/grpcio-1.82.1.tar.gz", hash = "sha256:707b24abd90fcb1e45bcc080577da1dbf9971d107490589b9539af8e1e77b4b5", size = 13187300, upload-time = "2026-07-08T12:36:16.588Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/88/d1350bf3343a2ed87d801584e40609f6c6bd3087926eeca03de50348cf4a/grpcio-1.82.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:c09bd5fa0d5b1fbd773ec349fe61441c3e4ebf168c229aa7538a820bdfad6a58", size = 6144689, upload-time = "2026-07-08T12:34:55.567Z" }, + { url = "https://files.pythonhosted.org/packages/e6/33/71875cdecd27c24ac1385d4783a09853f01b84a825a36aec2a2bc7d0d080/grpcio-1.82.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1eae24810720734598e3e6a1a528d5de0f265fe3fc86575e9ecce424b9ec7379", size = 11952034, upload-time = "2026-07-08T12:34:58.128Z" }, + { url = "https://files.pythonhosted.org/packages/82/b2/d9125df3d8a140dec12cc82c05b7deafedeababcff6496f28b2fd5634d10/grpcio-1.82.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a6bd5daf5bde7b24d7ad2cbaf8bf9eac620d96222016bb5e7ddde930dec0673f", size = 6710772, upload-time = "2026-07-08T12:35:01.33Z" }, + { url = "https://files.pythonhosted.org/packages/88/9b/69e2d1627398b964f34437dc476a5aff5a2cc8e7f247d26272b5674b5faf/grpcio-1.82.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1ecfde669cb687ac020d31ff76debe5dc7a62213335f02262eb6625628da1c03", size = 7450677, upload-time = "2026-07-08T12:35:03.926Z" }, + { url = "https://files.pythonhosted.org/packages/e4/e7/8f855ca29c294956122a2a73023655b9b02602d5111dad2b9b00e7631c68/grpcio-1.82.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:011c8badee95734dee8bf05ce3464756a0ac3ebb8d443afd20c0e2b5e4640ad9", size = 6886855, upload-time = "2026-07-08T12:35:06.174Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f0/fa87e85f49925f44c479d07e58b051e69bcfef6b6d5fbc6749d140f6730a/grpcio-1.82.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b85f4564926fb23114d239392bdcae200db1e6179629edd7d7ab0ab89c96a197", size = 7501323, upload-time = "2026-07-08T12:35:08.49Z" }, + { url = "https://files.pythonhosted.org/packages/67/55/2e0b10ae1d3ef9dcc480b91dc2158f4931fc4675d3af0a2836e39b2a744f/grpcio-1.82.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2c0c8270833395644c3fe6b6a806397955a2bc0538000a19a78b90c05a6c16e0", size = 8536899, upload-time = "2026-07-08T12:35:10.975Z" }, + { url = "https://files.pythonhosted.org/packages/bd/95/a3d8b0431fa221efc51ee39d73595ede74ba82a43b7c4313192e580face2/grpcio-1.82.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2ba199205ff46c7778290fe1673c91ac8e7e45678dd5c86e9e56fa33ec8788f6", size = 7913892, upload-time = "2026-07-08T12:35:13.944Z" }, + { url = "https://files.pythonhosted.org/packages/b8/92/f2651ec704d9852a56faef394775038afba435b50ce82ab2404d119c3355/grpcio-1.82.1-cp312-cp312-win32.whl", hash = "sha256:06127691866e295c14e84a1fb86356dd962254f6abd0da4ca4b001eea9e89438", size = 4240985, upload-time = "2026-07-08T12:35:16.048Z" }, + { url = "https://files.pythonhosted.org/packages/96/4f/a5fe8bf0d0a1b24855f370293075c931f27de4eb55f0f158786095bf3c11/grpcio-1.82.1-cp312-cp312-win_amd64.whl", hash = "sha256:1fa3223a3a2e1db74f4c2b255189eb7ea875dfba56e221d252ee3fc7b204778e", size = 5001580, upload-time = "2026-07-08T12:35:18.689Z" }, + { url = "https://files.pythonhosted.org/packages/1b/3e/496992d08c0aaa11272eb6228dc8ab947da01fe835de243cd00521bce4c4/grpcio-1.82.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:b454a2d97bfab7565683a02345f86bd182ab69fd7c2bdb7414171e7538f266b1", size = 6146068, upload-time = "2026-07-08T12:35:21.365Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8f/f263d6f14fdba6b56cfadd91fd3e158a52682b72c6016d1f8723d435659f/grpcio-1.82.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3dde70abfc80b3be11de53ba0d601c439e7fb2afd3583ad1788d1146bec92fdc", size = 11948600, upload-time = "2026-07-08T12:35:24.312Z" }, + { url = "https://files.pythonhosted.org/packages/8c/14/3a02e6ee49c2d85bc15eaae321e0e11ab3542cad3c5b2de121ecce0c4296/grpcio-1.82.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5523099c98c292ea1ae08e617249db760c56a78f8deae879027fe7d1ffbcbf6", size = 6714591, upload-time = "2026-07-08T12:35:27.027Z" }, + { url = "https://files.pythonhosted.org/packages/69/80/58e3738696f48ab7645347b98d8a7f93d10e00e6218388fbfcd6c9310e3d/grpcio-1.82.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5e5c4dc0a59b0f8490a6bdfd6fc8395b9d8ad8a8407c7d67ca7b5bba15c0877f", size = 7454995, upload-time = "2026-07-08T12:35:29.599Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6c/2557c1a889363072fbf2285ecd0e8c44860d4dbd60f017a32537c5b863e2/grpcio-1.82.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c40d94ba820329cc191981bc22fa6f6eed0799c6d921f3c6709521d59d4a2fd7", size = 6888621, upload-time = "2026-07-08T12:35:32.38Z" }, + { url = "https://files.pythonhosted.org/packages/d2/66/907706ccaff1223f1e10fd5b37fc16faead43392fccb4e786e7e390ac141/grpcio-1.82.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c816180e31e273caaec6f8bd86a8392499d5bbb26f41da44e3dce48bde69095", size = 7505069, upload-time = "2026-07-08T12:35:35.072Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7c/ff97b0d0f635987ee5ec80dfedafa1aad629303745d48e8637d10eec5b80/grpcio-1.82.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e31fd780b261830720cb70b0fd8f0aa51d49e75a66d7464ad2e31d4b765f2580", size = 8535384, upload-time = "2026-07-08T12:35:37.954Z" }, + { url = "https://files.pythonhosted.org/packages/62/9e/a97fddd970a8d1588cade06eca20443761c1858b0ad6590a5c835aa18062/grpcio-1.82.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d76152d7c31d7210d4a106e5d8b64da5bba5d6abf11be30e2f7b0a0c59bbcbf", size = 7910707, upload-time = "2026-07-08T12:35:40.797Z" }, + { url = "https://files.pythonhosted.org/packages/20/e4/eaba1517888af483a88d449eb7566f0f7f63446d46f339c5891798435875/grpcio-1.82.1-cp313-cp313-win32.whl", hash = "sha256:38e9dcb5258226fb3282630b31b16a968df52c8c6ad514af540646e0a4578f8a", size = 4240363, upload-time = "2026-07-08T12:35:43.298Z" }, + { url = "https://files.pythonhosted.org/packages/b0/42/66a98d47732e35290bef722f6149fed3709cd4cf61166f6f53a12f417302/grpcio-1.82.1-cp313-cp313-win_amd64.whl", hash = "sha256:3dbfb52c36d9511ac2b8e6c94fdde837b393ae520cc321f52a333a2deedf5a90", size = 5000980, upload-time = "2026-07-08T12:35:46.262Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cb/cf9ae9e164c6e6dc8a494faa9771763df9da150eefe19671009624d1559f/grpcio-1.82.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:35f990f7784c8fd2872644f07f96ebb4d9e48e145a190ab80d0280af91a1bfb2", size = 6146901, upload-time = "2026-07-08T12:35:49.261Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2a/eccf26dbcfb7f7cab8027c5490a16c8937c5aa7a2ec20a3eab2cf7a43165/grpcio-1.82.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:46536a4a1f4434df3c851b9254ff6fc7df5705b273681a15ca277d5921c178a0", size = 11954756, upload-time = "2026-07-08T12:35:52.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/75/3b3b4a3cc9f084b026af96e1d3e539b1af29ec7f41ed0dfff3cb99cc8626/grpcio-1.82.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d6650a7c1ebb7921c70e12a385439a8118efb99e669fa9ed31cf25db1843937c", size = 6723087, upload-time = "2026-07-08T12:35:54.973Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8b/b0f0c9b1400a99a4da4c09b114f101b192f8f11192e76f620b8962f5d90b/grpcio-1.82.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b8e110c66df5204c0506d6c8787b35d48b8b699ef5aa366d6c4d67325c67fe9a", size = 7454542, upload-time = "2026-07-08T12:35:57.586Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bd/428e38868382aa193697a5aa53973f29c58e58ba4268aa0c86a2715ee58b/grpcio-1.82.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f853eae07235a51a27bb5d6a9a175a59ca55dc9b99edc6ce2f76f07332d333ae", size = 6889588, upload-time = "2026-07-08T12:36:00.012Z" }, + { url = "https://files.pythonhosted.org/packages/49/ce/03e01d5e10259bf5c08ee50570cc94724e79c956f61fd2f09b341af0956c/grpcio-1.82.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:60b0f2c95337694fc094b77d9f60f50566c84b5677393e342eb98daeee242d98", size = 7514166, upload-time = "2026-07-08T12:36:02.693Z" }, + { url = "https://files.pythonhosted.org/packages/ff/59/278b4b600329e2ba3849f3c1ea3c820b3a01b38a7ad184ba09595e8d2733/grpcio-1.82.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:b064fc444812bdaa9825d33c26f8d732d63ee6a5d78557c1faf92c98687fed27", size = 8536166, upload-time = "2026-07-08T12:36:05.349Z" }, + { url = "https://files.pythonhosted.org/packages/44/27/7ccf2ef00f27a8e47a79d641c8ceaf7d3028c7a03d9a97b4c8a9a783c086/grpcio-1.82.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d7ede11d747b4e1bd05e3bc0260e155b65a88735a895a10f6521f19b889511e", size = 7912572, upload-time = "2026-07-08T12:36:08.393Z" }, + { url = "https://files.pythonhosted.org/packages/0d/be/33742482d2753f2d3a1b7641664b6622262d44f2f3b609f13425dd86d36f/grpcio-1.82.1-cp314-cp314-win32.whl", hash = "sha256:3d21f19838dc255ecbb79321b15ae9b98fbddff4c3d4aedb0a81bdd7f4ab572a", size = 4321856, upload-time = "2026-07-08T12:36:10.899Z" }, + { url = "https://files.pythonhosted.org/packages/cc/67/03329c847172c78ddeb1eb9be6b444fdbc12775a84c958b27e427e7b926d/grpcio-1.82.1-cp314-cp314-win_amd64.whl", hash = "sha256:e20f1edbb15f99e3128ec86433f9785fd5a451d8f115e74fe0056134f092a9d5", size = 5141114, upload-time = "2026-07-08T12:36:13.595Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1260,6 +1339,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "milvus-lite" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "faiss-cpu" }, + { name = "grpcio" }, + { name = "numpy" }, + { name = "pyarrow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/9a/d80d260e6fe1246818a8ef782c374ba9c6ca46ca3b987c14eabe914ef805/milvus_lite-3.0.tar.gz", hash = "sha256:2c35d0d046b1faae3402cde1fb73d65f51ee8c6aba65f54de1dda46f7bb18b9b", size = 589749, upload-time = "2026-05-13T07:14:05.827Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/42/dee08ae3bfc1731572f193d00b248c9370b0b9dff12becb0ffd8b2ee8d56/milvus_lite-3.0-py3-none-any.whl", hash = "sha256:d9a094eab84bdaa4253da3721482282c939da1cce6f4e1759f947e8d3e53406e", size = 230490, upload-time = "2026-05-13T07:14:00.816Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -1339,6 +1433,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/1c/5d43735b2553baae2a5e899dcbcd0670a86930d993184d72ca909bf11c9b/openai-2.36.0-py3-none-any.whl", hash = "sha256:143f6194b548dbc2c921af1f1b03b9f14c85fed8a75b5b516f5bcc11a2a50c63", size = 1302361, upload-time = "2026-05-07T17:33:15.063Z" }, ] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -1348,6 +1495,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09", size = 10357846, upload-time = "2026-05-11T18:52:36.143Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", size = 9899550, upload-time = "2026-05-11T18:52:38.976Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/fa2535168fffcedf67f4f6de28d2dd903a747ca7c8ea6989451aaeb3a92f/pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", size = 10412965, upload-time = "2026-05-11T18:52:41.915Z" }, + { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a4/2eb28f2fccb4ced4a2c79ab2a5dee9ade1ebf44922ebad6fea158c9f95d4/pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", size = 11422824, upload-time = "2026-05-11T18:52:48.058Z" }, + { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c5/fc1b368f303087d20e8c9bf3d6ceb186263cfac0ade735cd938538bea839/pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", size = 9755463, upload-time = "2026-05-11T18:52:53.386Z" }, + { url = "https://files.pythonhosted.org/packages/86/bd/fda8f9705b1b09c6ebe14bfc0fa0e4ec8584d54ea673628f157ff55131af/pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", size = 9066158, upload-time = "2026-05-11T18:52:56.038Z" }, + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, +] + [[package]] name = "parso" version = "0.8.7" @@ -1505,6 +1704,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, ] +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + [[package]] name = "psutil" version = "7.2.2" @@ -1756,6 +1970,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/97/03635143a12a5d941f545548b00f8ac39d35565321a2effb4154ed267338/pyinstrument-5.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:b6a71f5e7f53c86c9b476b30cf19509463a63581ef17ddbd8680fee37ae509db", size = 128164, upload-time = "2026-01-04T18:38:32.281Z" }, ] +[[package]] +name = "pymilvus" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "grpcio" }, + { name = "orjson" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "setuptools" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/c1/01647e61f3a82fd881382746b6dde3401d65b88cd4f75bd059901fb2392b/pymilvus-3.0.0-1-py3-none-any.whl", hash = "sha256:57c8e7c87fbbf579f122b4df893949bc78e50bca2988527864891bd544817b05", size = 344817, upload-time = "2026-05-07T14:57:45.235Z" }, +] + +[package.optional-dependencies] +milvus-lite = [ + { name = "milvus-lite", marker = "sys_platform != 'win32'" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -2061,6 +2298,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" From 64161417bc2444dd1cb9b5bf9d82ea9b9890018d Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 9 Jul 2026 10:33:00 +0000 Subject: [PATCH 2/3] fix(milvus): bound fact recall and use exact counts --- .../infra/persistence/milvus/repository.py | 12 ++++++-- .../memory/search/recall/atomic_fact.py | 6 +++- .../unit/test_infra/test_milvus/test_repo.py | 1 + .../test_search/test_recall_atomic_fact.py | 29 +++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index b8828118e..90019f414 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -167,9 +167,15 @@ async def rebuild_indexes(self) -> None: async def count(self) -> int: await self.ensure_collection() client = await get_client() - stats = await _run(client.get_collection_stats, self.collection_name) - row_count = stats.get("row_count", 0) - return int(row_count) + rows = await _run( + client.query, + self.collection_name, + filter="", + output_fields=["count(*)"], + ) + if not rows: + return 0 + return int(rows[0].get("count(*)", 0)) async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: if id_field != "id": diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 38ec3872a..f78b3f50a 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -32,6 +32,7 @@ _NOISE_COLUMNS = frozenset( {"vector", "_distance", "_score", "created_at", "updated_at"} ) +_MAX_FACT_RECALL_LIMIT = 1024 class AtomicFactRecaller: @@ -160,7 +161,10 @@ async def _query_facts_for_parents( quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) clause = f"parent_id IN ({quoted})" full_where = f"({where}) AND ({clause})" - limit = per_episode * max(len(parent_to_eps), 1) + # Milvus server / Zilliz Cloud reject search topK values above 1024. + # The fact expansion only needs a bounded candidate pool for top-N + # competition, so keep the same cap for every derived-index backend. + limit = min(per_episode * max(len(parent_to_eps), 1), _MAX_FACT_RECALL_LIMIT) if query_vector: return await atomic_fact_repo.dense_search( query_vector, full_where, limit=limit diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 901d67034..8d265d81f 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -109,6 +109,7 @@ async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: rows = await episode_repo.find_where(filters, limit=10) assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} assert rows[0].timestamp.utcoffset() == dt.timedelta(0) + assert await episode_repo.count() == 2 sparse = await episode_repo.sparse_search( ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index f2e80b88e..c3c71d433 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -29,6 +29,7 @@ atomic_fact_repo, lancedb_manager, ) +from everos.memory.search.recall import atomic_fact as atomic_fact_module from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -267,6 +268,34 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: assert out["alice_ep_a"][0].score == 0.0 +async def test_facts_for_episodes_caps_dense_recall_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Large parent fan-out must not exceed Milvus/Zilliz search topK limits.""" + captured: dict[str, int] = {} + + async def fake_dense_search(vector, where, *, limit): # type: ignore[no-untyped-def] + captured["limit"] = limit + return [] + + monkeypatch.setattr( + atomic_fact_module.atomic_fact_repo, + "dense_search", + fake_dense_search, + ) + ep_to_parents = {f"alice_ep_{i}": [f"parent_{i}"] for i in range(80)} + + out = await _recaller().facts_for_episodes( + ep_to_parents, + "owner_id = 'alice' AND owner_type = 'user'", + per_episode=20, + query_vector=_unit_vector(0), + ) + + assert out == {} + assert captured["limit"] == 1024 + + # ── Dual parent_id (post-1.5 migration) ──────────────────────────────── From 708e8e97887d68cff3032a1504d176acc13b77dc Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Fri, 10 Jul 2026 03:39:14 +0000 Subject: [PATCH 3/3] style: format Milvus index changes --- src/everos/config/settings.py | 4 +--- src/everos/infra/persistence/index/__init__.py | 4 +--- src/everos/infra/persistence/milvus/repository.py | 6 ++++-- src/everos/memory/search/filters.py | 4 +--- tests/unit/test_memory/test_search/test_filters.py | 6 +++--- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 8d5c593ba..5f9544491 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -365,9 +365,7 @@ class MilvusSettings(BaseModel): uri: str | None = None token: SecretStr | None = None db_name: str | None = None - consistency_level: Literal["Strong", "Session", "Bounded", "Eventually"] = ( - "Session" - ) + consistency_level: Literal["Strong", "Session", "Bounded", "Eventually"] = "Session" dimension: int = Field(default=1024, ge=1) collection_prefix: str = "everos" diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index 486900058..0cc969eb6 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -278,9 +278,7 @@ def _milvus() -> Any: atomic_fact_repo = _IndexRepoRouter( _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact ) -foresight_repo = _IndexRepoRouter( - _lancedb.foresight_repo, "foresight_repo", Foresight -) +foresight_repo = _IndexRepoRouter(_lancedb.foresight_repo, "foresight_repo", Foresight) agent_case_repo = _IndexRepoRouter( _lancedb.agent_case_repo, "agent_case_repo", AgentCase ) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index 90019f414..de9143b3d 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -550,8 +550,10 @@ def _restore_row( out[name] = from_timestamp(int(value)) elif name in row: value = row[name] - if name in self.schema.BM25_FIELDS and value == "" and _is_optional( - field.annotation + if ( + name in self.schema.BM25_FIELDS + and value == "" + and _is_optional(field.annotation) ): out[name] = None else: diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 22df21a47..0f417afc0 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -190,9 +190,7 @@ def _compile_filters_backend( # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": base.append( - "deprecated_by IS NULL" - if backend == "lancedb" - else "deprecated_by is null" + "deprecated_by IS NULL" if backend == "lancedb" else "deprecated_by is null" ) if node is None: return " AND ".join(base) diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index a4267d8c8..59e85dd2e 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -276,7 +276,7 @@ def test_compile_filters_for_backends_preserves_lancedb_default() -> None: filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") assert str(filters) == filters.lancedb assert "owner_id = 'u_a'" in filters.lancedb - assert "owner_id == \"u_a\"" in filters.milvus + assert 'owner_id == "u_a"' in filters.milvus assert "deprecated_by IS NULL" in filters.lancedb assert "deprecated_by is null" in filters.milvus @@ -295,7 +295,7 @@ def test_compile_filters_for_milvus_timestamp_and_array() -> None: def test_compile_filters_for_milvus_escapes_string_literals() -> None: node = FilterNode.model_validate({"session_id": "ses's"}) filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") - assert "owner_id == \"al'ice\"" in filters.milvus - assert "session_id == \"ses's\"" in filters.milvus + assert 'owner_id == "al\'ice"' in filters.milvus + assert 'session_id == "ses\'s"' in filters.milvus assert "owner_id = 'al''ice'" in filters.lancedb assert "session_id = 'ses''s'" in filters.lancedb
Local three-part stack✅ Markdown + SQLite + LanceDB; no MongoDB, Elasticsearch, or Redis required✅ Markdown + SQLite + embedded LanceDB by default; optional Milvus when configured ❌ Often depends on managed services, vector DBs, graph DBs, or server stacks