diff --git a/.github/workflows/loctree-ci.yml b/.github/workflows/loctree-ci.yml index c616c7d..476f527 100644 --- a/.github/workflows/loctree-ci.yml +++ b/.github/workflows/loctree-ci.yml @@ -1,7 +1,7 @@ # Loctree CI - Codebase Health Analysis # Uses loctree 0.8.0+ for dead code detection, circular imports, and health scoring # -# Co-Authored-By: Maciej & Klaudiusz +# Copyright (c) 2025-2026 Loctree (https://loct.io) name: Loctree CI diff --git a/.vibecrafted/plans b/.vibecrafted/plans deleted file mode 120000 index 5e00b20..0000000 --- a/.vibecrafted/plans +++ /dev/null @@ -1 +0,0 @@ -/Users/polyversai/.vibecrafted/artifacts/VetCoders/rmcp-memex/2026_0408/marbles/plans \ No newline at end of file diff --git a/.vibecrafted/reports b/.vibecrafted/reports deleted file mode 120000 index d843b97..0000000 --- a/.vibecrafted/reports +++ /dev/null @@ -1 +0,0 @@ -/Users/polyversai/.vibecrafted/artifacts/VetCoders/rmcp-memex/2026_0408/marbles/reports \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 38e4a1f..5d97ec8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### Changed - **Rebranded: `rmcp-memex` → `rust-memex`.** Crate name on crates.io, binary name `rmcp-memex` → `rust-memex`, library path `rmcp_memex` → `rust_memex`, and all internal imports `use rmcp_memex::` → `use rust_memex::`. User-facing `RMCP_MEMEX_*` environment variables preserved as-is for backward compatibility. - **Moved to Loctree org:** `https://github.com/VetCoders/rmcp-memex` → `https://github.com/Loctree/rust-memex`. Docs, install CTAs, and issue tracker URLs updated across README, docs/, install.sh, and Makefile. -- **Authors + license unchanged.** Maciej Gad & Monika Szymanska remain the Cargo authors; dual MIT OR Apache-2.0 license retained for library-friendly downstream use. +- **License:** the crate ships under the Business Source License 1.1 (BUSL-1.1); see `LICENSE` for terms and the Apache-2.0 change license. ### Added - `rust-memex reindex` subcommand for rebuilding namespace contents with paginated source reads, preflight embedding checks, and target-namespace safety guards @@ -171,7 +171,7 @@ All notable changes to this project will be documented in this file. - **CLI `--auto-route` flag** - Automatic search mode selection for `search` command - Analyzes query intent and selects optimal mode (vector/bm25/hybrid) - Displays intent, confidence, and loctree suggestions when applicable - - Example: `rust-memex search -n memories -q "when did we buy dragon" --auto-route` + - Example: `rust-memex search -n memories -q "when did we buy the laptop" --auto-route` - **MCP `auto_route` parameter** - Added to `rag_search` and `memory_search` tools - When `true`, QueryRouter overrides explicit `mode` parameter - Enables intelligent mode selection for AI agents @@ -216,7 +216,7 @@ All notable changes to this project will be documented in this file. ### Added - **MemexEngine** - High-level API for library consumers - `MemexEngine::for_app()` - quick setup for any application with auto-config - - `MemexEngine::for_vista()` - Vista-optimized defaults (1024 dims, qwen3-embedding:0.6b) + - `MemexEngine::for_app_optimized()` - optimized defaults (1024 dims, qwen3-embedding:0.6b) - CRUD operations: `store()`, `search()`, `get()`, `delete()` - Batch operations: `store_batch()` for efficient bulk inserts - Filtered operations: `search_filtered()`, `delete_by_filter()` for GDPR-compliant deletion @@ -370,4 +370,4 @@ All notable changes to this project will be documented in this file. --- -Vibecrafted with AI Agents by Loctree (c)2025 The LibraxisAI Team +Vibecrafted with AI Agents by Loctree (c)2025 Vetcoders diff --git a/Cargo.toml b/Cargo.toml index f0c5994..adf41d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.6.0" edition = "2024" rust-version = "1.88" description = "Operator CLI + MCP server: canonical corpus second: semantic index second to aicx" -authors = ["Maciej Gad ", "Monika Szymanska "] +authors = ["Libraxis AI sp. z o.o. "] license = "BUSL-1.1" repository = "https://github.com/Loctree/rust-memex" homepage = "https://github.com/Loctree/rust-memex" diff --git a/LICENSE b/LICENSE index b1b688b..0ad1d6d 100644 --- a/LICENSE +++ b/LICENSE @@ -3,24 +3,24 @@ License text copyright (c) 2020 MariaDB Corporation Ab, All Rights Reserved. Parameters -Licensor: VetCoders (Maciej Gad & Monika Szymanska) +Licensor: Libraxis AI sp. z o.o. Licensed Work: Rust Memex - The Intentions Kernel. - The Licensed Work is (c) 2024-2026 VetCoders. + The Licensed Work is (c) 2024-2026 Libraxis AI sp. z o.o. Additional Use Grant: You may make production use of the Licensed Work, provided Your use does not include offering the Licensed Work to third parties on a hosted or embedded - basis in order to compete with VetCoders' paid + basis in order to compete with Libraxis AI's paid version(s) of the Licensed Work. For purposes of this license: A "competitive offering" is a Product that is offered to third parties on a paid basis, including through paid support arrangements, that significantly overlaps - with the capabilities of VetCoders' paid version(s) of + with the capabilities of Libraxis AI's paid version(s) of the Licensed Work. If Your Product is not a competitive offering when You first make it generally available, it will not become a competitive offering later due to - VetCoders releasing a new version of the Licensed Work + Libraxis AI releasing a new version of the Licensed Work with additional capabilities. In addition, Products that are not provided on a paid basis are not competitive. @@ -48,7 +48,7 @@ Change Date: 2030-03-24 Change License: Apache License, Version 2.0 For information about alternative licensing arrangements for the Licensed -Work, please contact void@div0.space. +Work, please contact contact@libraxis.ai. Notice diff --git a/Makefile b/Makefile index cc44a4d..4bd27a4 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ # rust-memex Makefile # ============================================================================ # Service management, build, and maintenance targets -# Created by M&K (c)2026 Loctree +# Created by Vetcoders (c)2026 Loctree # ============================================================================ # -# RAM DISK MODE (Dragon 512GB): +# RAM DISK MODE (512GB host): # make ramdisk-up - Create 50GB RAM disk, copy DB, start service # make ramdisk-down - Sync to disk, unmount RAM disk, stop service # make snapshot - Sync RAM disk to disk (backup) @@ -14,7 +14,7 @@ SHELL := /bin/bash BINARY := rust-memex INSTALL_PATH := $(HOME)/.cargo/bin/$(BINARY) -LAUNCHD_PLIST := $(HOME)/Library/LaunchAgents/ai.libraxis.rust-memex.plist +LAUNCHD_PLIST := $(HOME)/Library/LaunchAgents/ai.vetcoders.rust-memex.plist # Disk paths DB_PATH_DISK := $(HOME)/.ai-memories/lancedb @@ -63,14 +63,14 @@ start: ## Start memex service via launchd echo "Service already running on port $(HTTP_PORT)"; \ else \ launchctl bootstrap gui/$$(id -u) $(LAUNCHD_PLIST) 2>/dev/null || \ - launchctl kickstart gui/$$(id -u)/ai.libraxis.rust-memex 2>/dev/null || \ + launchctl kickstart gui/$$(id -u)/ai.vetcoders.rust-memex 2>/dev/null || \ $(INSTALL_PATH) serve --db-path $(DB_PATH) --http-port $(HTTP_PORT) --http-only & \ sleep 3; \ echo "Started memex on port $(HTTP_PORT)"; \ fi stop: ## Stop memex service - @-launchctl bootout gui/$$(id -u)/ai.libraxis.rust-memex 2>/dev/null + @-launchctl bootout gui/$$(id -u)/ai.vetcoders.rust-memex 2>/dev/null @-pkill -f "$(BINARY) serve" 2>/dev/null @echo "Stopped memex service" @@ -105,7 +105,7 @@ dashboard: ## Open dashboard in browser @open http://localhost:$(HTTP_PORT)/ # ============================================================================ -# RAM DISK (Dragon 512GB - full DB in RAM) +# RAM DISK (512GB host - full DB in RAM) # ============================================================================ ramdisk-create: ## Create 50GB RAM disk (requires sudo for mount) diff --git a/README.md b/README.md index 2035b19..a852f29 100644 --- a/README.md +++ b/README.md @@ -126,12 +126,12 @@ async fn main() -> anyhow::Result<()> { // Store a document engine.store( "doc-1", - "Patient presented with lethargy and decreased appetite", - json!({"patient_id": "P-123", "visit_type": "checkup"}) + "Onboarding notes: new users need a guided setup walkthrough", + json!({"source": "support", "topic": "onboarding"}) ).await?; // Search semantically - let results = engine.search("lethargy symptoms", 10).await?; + let results = engine.search("user onboarding setup", 10).await?; for r in &results { println!("{}: {} (score: {:.2})", r.id, r.text, r.score); } @@ -148,21 +148,21 @@ async fn main() -> anyhow::Result<()> { } ``` -### Vista Integration +### Optimized Profile -For Vista PIMS, use the optimized constructor: +For a smaller-footprint setup, use the optimized constructor: ```rust use rust_memex::MemexEngine; -// Vista-optimized: 1024 dims, qwen3-embedding:0.6b model -let engine = MemexEngine::for_vista().await?; +// Optimized: 1024 dims, qwen3-embedding:0.6b model +let engine = MemexEngine::for_app_optimized().await?; -// Store visit notes +// Store notes engine.store( - "visit-456", - "SOAP note: Feline diabetes mellitus diagnosis...", - json!({"patient_id": "P-789", "doc_type": "soap_note"}) + "note-456", + "Release notes: summary of shipped features and known issues...", + json!({"doc_type": "release_note", "version": "1.2.0"}) ).await?; ``` @@ -205,7 +205,7 @@ use rust_memex::{MemexEngine, SearchMode}; let engine = MemexEngine::for_app("my-app", "documents").await?; // Hybrid search with BM25 + vector fusion (recommended) -let results = engine.search_hybrid("dragon mac studio", 10).await?; +let results = engine.search_hybrid("fast mac studio", 10).await?; for r in &results { println!("{}: {} (combined: {:.2}, vector: {:.2}, bm25: {:.2})", r.id, r.document, r.combined_score, r.vector_score, r.bm25_score); @@ -309,9 +309,8 @@ MEMEX_BM25_PATH=~/.rmcp-servers/myapp/bm25 # ADVANCED: Multiple providers (fallback cascade) # ============================================================================= -# Remote embedding server fallback -# DRAGON_BASE_URL=http://your-server.local -# DRAGON_EMBEDDER_PORT=12345 +# Remote embedding server fallback (uses EMBEDDER_PORT below) +# EMBEDDER_BASE_URL=http://your-server.local # MLX embedder for Apple Silicon # EMBEDDER_PORT=12300 @@ -613,15 +612,15 @@ curl -X POST http://localhost:8997/upsert \ # Search curl -X POST http://localhost:8997/search \ -H "Content-Type: application/json" \ - -d '{"query": "context", "namespace": "agent1", "k": 10, "project": "Vista", "deep": true}' + -d '{"query": "context", "namespace": "agent1", "k": 10, "project": "Demo", "deep": true}' # SSE streaming search -curl -N "http://localhost:8997/sse/search?query=context&namespace=agent1&limit=5&project=Vista&layer=1" +curl -N "http://localhost:8997/sse/search?query=context&namespace=agent1&limit=5&project=Demo&layer=1" ``` ### Multi-Host Database Paths -For setups with multiple machines (e.g., dragon, mgbook16), use per-host database paths: +For setups with multiple machines (e.g., laptop, workstation), use per-host database paths: ```bash # Per-host paths (each machine gets own database) @@ -633,7 +632,7 @@ rust-memex wizard The TUI wizard auto-detects hostname and offers: - **Shared mode**: `~/.ai-memories/lancedb` (same path everywhere) -- **Per-host mode**: `~/.ai-memories/lancedb.dragon`, `~/.ai-memories/lancedb.mgbook16`, etc. +- **Per-host mode**: `~/.ai-memories/lancedb.host-a`, `~/.ai-memories/lancedb.host-b`, etc. ### Configuration (TOML) @@ -685,7 +684,7 @@ Intelligent query intent detection for automatic search mode selection: ```bash # Auto-detect query intent and select optimal mode -rust-memex search -n memories -q "when did we buy dragon" --auto-route +rust-memex search -n memories -q "when did we buy the laptop" --auto-route # Output: Query intent: temporal (confidence: 0.70) # Selects: hybrid mode with date boosting @@ -695,7 +694,7 @@ rust-memex search -n code -q "who imports main.rs" --auto-route # Consider: loctree query --kind who-imports --target main.rs # Deep exploration with all onion layers -rust-memex dive -n memories -q "dragon" --verbose +rust-memex dive -n memories -q "laptop" --verbose ``` **Intent Types:** @@ -817,5 +816,4 @@ Add to `~/.claude.json`: --- -Vibecrafted with AI Agents by Loctree (c)2025 The LibraxisAI Team -Co-Authored-By: [Maciej](void@div0.space) & [Klaudiusz](the1st@whoai.am) +Vibecrafted with AI Agents by Loctree (c)2025 Vetcoders diff --git a/SECURITY.md b/SECURITY.md index bad91b5..b734424 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,7 +13,7 @@ Please use GitHub Security Advisories for responsible disclosure: https://github.com/Loctree/rust-memex/security/advisories/new -If GitHub Advisories are unavailable, email security@loct.io with a +If GitHub Advisories are unavailable, email security@loctree.com with a summary, affected version, reproduction steps, and expected impact. Please do not report suspected vulnerabilities in public GitHub issues. diff --git a/ai-hooks/README.md b/ai-hooks/README.md index 8896fa9..a74d39f 100644 --- a/ai-hooks/README.md +++ b/ai-hooks/README.md @@ -48,4 +48,4 @@ Add to `~/.claude/settings.json`: --- -Vibecrafted with AI Agents by Loctree (c)2026 The LibraxisAI Team +Vibecrafted with AI Agents by Loctree (c)2026 Vetcoders diff --git a/ai-hooks/memex-context.sh b/ai-hooks/memex-context.sh index 36a50c9..2b13ccc 100755 --- a/ai-hooks/memex-context.sh +++ b/ai-hooks/memex-context.sh @@ -33,7 +33,7 @@ export PATH="$HOME/.cargo/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PA # - jq for JSON parsing (optional) # # ============================================================================ -# Created by M&K (c)2026 The LibraxisAI Team +# Created by Vetcoders (c)2026 # ============================================================================ set -uo pipefail diff --git a/ai-hooks/memex-startup.sh b/ai-hooks/memex-startup.sh index 67c5eee..29d99b4 100755 --- a/ai-hooks/memex-startup.sh +++ b/ai-hooks/memex-startup.sh @@ -17,7 +17,7 @@ export PATH="$HOME/.cargo/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PA # - SessionStart hook # # ============================================================================ -# Created by M&K (c)2026 The LibraxisAI Team +# Created by Vetcoders (c)2026 # ============================================================================ set -uo pipefail diff --git a/docs/01_security.md b/docs/01_security.md index 5101353..99e9de0 100644 --- a/docs/01_security.md +++ b/docs/01_security.md @@ -2,39 +2,45 @@ ## Problem -W środowisku multi-agent, gdzie wiele AI agentów może korzystać z tego samego serwera rust-memex, potrzebna jest izolacja danych między agentami. Bez mechanizmu kontroli dostępu: -- Agent A może odczytać dane Agenta B -- Brak możliwości ochrony wrażliwych namespace'ów -- Trudność w audycie dostępu do danych +In a multi-agent environment, where many AI agents may use the same +rust-memex server, data isolation between agents is required. Without an +access-control mechanism: +- Agent A can read Agent B's data +- There is no way to protect sensitive namespaces +- Auditing data access is difficult -Dodatkowo, ograniczenie dostępu do plików tylko do `$HOME` i `cwd` było zbyt restrykcyjne - blokowało legitymowe użycie zewnętrznych wolumenów (np. `/Volumes/ExternalDrive`). +In addition, restricting file access to only `$HOME` and `cwd` was too +restrictive - it blocked legitimate use of external volumes (e.g. +`/Volumes/ExternalDrive`). -## Rozwiązanie +## Solution -Zaimplementowano dwupoziomowy system bezpieczeństwa: +A two-level security system was implemented: -### 1. Konfigurowalna Whitelist Ścieżek +### 1. Configurable Path Whitelist -Zamiast hardcoded ograniczenia do `$HOME` i `cwd`, wprowadzono konfigurowalną listę dozwolonych ścieżek. +Instead of a hardcoded restriction to `$HOME` and `cwd`, a configurable +list of allowed paths was introduced. ```toml # ~/.rmcp-servers/rust-memex/config.toml allowed_paths = [ "~", # Home directory - "/Volumes/LibraxisShare/data", # External volume + "/Volumes/ExternalDrive/data", # External volume "/opt/shared/documents" # Shared directory ] ``` -**Zachowanie:** -- Jeśli `allowed_paths` jest puste → domyślnie `$HOME` + `cwd` (backward compatible) -- Jeśli `allowed_paths` jest ustawione → tylko te ścieżki są dozwolone -- Wspiera `~` expansion do home directory -- Walidacja przez canonicalization (rozwiązuje symlinki) +**Behavior:** +- If `allowed_paths` is empty → defaults to `$HOME` + `cwd` (backward compatible) +- If `allowed_paths` is set → only those paths are allowed +- Supports `~` expansion to the home directory +- Validation via canonicalization (resolves symlinks) ### 2. Namespace Access Tokens -Token-based access control dla namespace'ów. Chronione namespace'y wymagają tokena do odczytu/zapisu. +Token-based access control for namespaces. Protected namespaces require a +token for read/write. ``` ┌─────────────────────────────────────────────────────────────┐ @@ -43,7 +49,7 @@ Token-based access control dla namespace'ów. Chronione namespace'y wymagają to │ │ │ Public Namespace Protected Namespace │ │ ┌─────────────┐ ┌─────────────────────┐ │ -│ │ "default" │ │ "pamietnik" │ │ +│ │ "default" │ │ "diary" │ │ │ │ │ │ │ │ │ │ No token │ │ Token: rmx_7f3a9b │ │ │ │ required │ │ required │ │ @@ -55,20 +61,20 @@ Token-based access control dla namespace'ów. Chronione namespace'y wymagają to └─────────────────────────────────────────────────────────────┘ ``` -## Użycie +## Usage -### Włączenie Security +### Enabling Security ```bash # CLI rust-memex serve --security-enabled -# Lub w config.toml +# Or in config.toml security_enabled = true token_store_path = "~/.rmcp-servers/rust-memex/tokens.json" ``` -### Tworzenie Tokena dla Namespace +### Creating a Token for a Namespace ```json // MCP Request @@ -77,7 +83,7 @@ token_store_path = "~/.rmcp-servers/rust-memex/tokens.json" "params": { "name": "namespace_create_token", "arguments": { - "namespace": "pamietnik", + "namespace": "diary", "description": "Personal diary namespace" } } @@ -87,37 +93,38 @@ token_store_path = "~/.rmcp-servers/rust-memex/tokens.json" { "content": [{ "type": "text", - "text": "Token created for namespace 'pamietnik': rmx_a1b2c3d4e5f6..." + "text": "Token created for namespace 'diary': rmx_a1b2c3d4e5f6..." }] } ``` -**WAŻNE:** Token jest zwracany tylko raz przy tworzeniu. Zapisz go w bezpiecznym miejscu! +**IMPORTANT:** The token is returned only once, at creation time. Store it +in a safe place! -### Dostęp do Chronionego Namespace +### Accessing a Protected Namespace ```json -// Bez tokena - BŁĄD +// Without a token - ERROR { "method": "tools/call", "params": { "name": "memory_search", "arguments": { - "namespace": "pamietnik", - "query": "moje wspomnienia" + "namespace": "diary", + "query": "my memories" } } } -// Error: "Access denied: namespace 'pamietnik' requires a valid token" +// Error: "Access denied: namespace 'diary' requires a valid token" -// Z tokenem - OK +// With a token - OK { "method": "tools/call", "params": { "name": "memory_search", "arguments": { - "namespace": "pamietnik", - "query": "moje wspomnienia", + "namespace": "diary", + "query": "my memories", "token": "rmx_a1b2c3d4e5f6..." } } @@ -125,7 +132,7 @@ token_store_path = "~/.rmcp-servers/rust-memex/tokens.json" // Success: returns search results ``` -### Odwołanie Tokena +### Revoking a Token ```json { @@ -133,15 +140,15 @@ token_store_path = "~/.rmcp-servers/rust-memex/tokens.json" "params": { "name": "namespace_revoke_token", "arguments": { - "namespace": "pamietnik" + "namespace": "diary" } } } ``` -Po odwołaniu tokena, namespace staje się ponownie publiczny. +After a token is revoked, the namespace becomes public again. -### Lista Chronionych Namespace'ów +### Listing Protected Namespaces ```json { @@ -156,12 +163,12 @@ Po odwołaniu tokena, namespace staje się ponownie publiczny. { "content": [{ "type": "text", - "text": "[\"pamietnik\", \"projekty\", \"finanse\"]" + "text": "[\"diary\", \"projects\", \"finance\"]" }] } ``` -### Status Security +### Security Status ```json { @@ -189,32 +196,32 @@ Po odwołaniu tokena, namespace staje się ponownie publiczny. } ``` -## Format Tokena +## Token Format -Tokeny mają format: `rmx_<32 znaki hex>` +Tokens have the format: `rmx_<32 hex chars>` ``` rmx_a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6 │ └─────────────────────────────────┘ -│ 32 znaki hex (128 bit) +│ 32 hex chars (128 bit) └── prefix "rmx_" (rust-memex) ``` -Tokeny są generowane kryptograficznie bezpiecznie (`rand::thread_rng()`). +Tokens are generated cryptographically securely (`rand::thread_rng()`). ## Token Store -Tokeny są przechowywane w pliku JSON: +Tokens are stored in a JSON file: ```json // ~/.rmcp-servers/rust-memex/tokens.json { - "pamietnik": { + "diary": { "token_hash": "5e884898da28047d...", "created_at": "2024-12-22T10:30:00Z", "description": "Personal diary namespace" }, - "projekty": { + "projects": { "token_hash": "d033e22ae348aeb5...", "created_at": "2024-12-22T11:00:00Z", "description": null @@ -222,66 +229,66 @@ Tokeny są przechowywane w pliku JSON: } ``` -**Bezpieczeństwo:** -- Przechowywany jest tylko **hash** tokena (SHA-256), nie sam token -- Token plaintext jest zwracany tylko raz przy tworzeniu -- Weryfikacja przez porównanie hashy +**Security:** +- Only the **hash** of the token (SHA-256) is stored, not the token itself +- The plaintext token is returned only once, at creation time +- Verification is done by comparing hashes ## Path Validation -Funkcja `validate_path()` chroni przed path traversal attacks: +The `validate_path()` function protects against path traversal attacks: ```rust -// Blokowane wzorce +// Blocked patterns "../../../etc/asdpasswd" // Path traversal sd"/etc/passwd" // Outside allowed paths "~/../../root/.ssh" // Traversal after expansion -// Dozwolone (jeśli w allowed_paths) +// Allowed (if in allowed_paths) "~/Documents/notes.md" // Under home "/Volumes/Data/file.txt" // Configured external volume ``` -**Walidacja:** -1. Sprawdzenie czy ścieżka nie jest pusta -2. Expansion `~` do home directory -3. Sprawdzenie wzorca `..` (path traversal) -4. Canonicalization (rozwiązanie symlinków) -5. Sprawdzenie czy canonical path jest pod dozwoloną ścieżką +**Validation:** +1. Check that the path is not empty +2. Expand `~` to the home directory +3. Check for the `..` pattern (path traversal) +4. Canonicalization (resolve symlinks) +5. Check that the canonical path is under an allowed path -## Implementacja +## Implementation -### Pliki źródłowe +### Source Files -| Plik | Opis | +| File | Description | |------|------| | `rust-memex/src/security/mod.rs` | `NamespaceAccessManager`, `TokenStore`, token generation/verification | -| `rust-memex/src/handlers/mod.rs` | `validate_path()`, integration z access manager | +| `rust-memex/src/handlers/mod.rs` | `validate_path()`, integration with the access manager | | `rust-memex/src/lib.rs` | `NamespaceSecurityConfig`, re-exports | | `rust-memex/src/bin/rust-memex.rs` | CLI flags `--security-enabled`, `--token-store-path` | -### Kluczowe struktury +### Key Structures ```rust -/// Konfiguracja security +/// Security configuration pub struct NamespaceSecurityConfig { pub enabled: bool, pub token_store_path: Option, } -/// Manager dostępu do namespace'ów +/// Namespace access manager pub struct NamespaceAccessManager { enabled: bool, store: Option>>, } -/// Przechowywanie tokenów +/// Token storage pub struct TokenStore { path: PathBuf, tokens: HashMap, } -/// Wpis tokena +/// Token entry pub struct TokenEntry { pub token_hash: String, pub created_at: DateTime, @@ -289,50 +296,49 @@ pub struct TokenEntry { } ``` -### Testy +### Tests ```bash cd rust-memex && cargo test security ``` -Testy pokrywają: -- `test_token_generation` - generowanie tokenów w poprawnym formacie -- `test_access_manager_disabled` - zachowanie gdy security wyłączone -- `test_token_store_create_and_verify` - tworzenie i weryfikacja tokenów -- `test_access_manager_enabled` - pełny flow z włączonym security +The tests cover: +- `test_token_generation` - generating tokens in the correct format +- `test_access_manager_disabled` - behavior when security is disabled +- `test_token_store_create_and_verify` - creating and verifying tokens +- `test_access_manager_enabled` - full flow with security enabled ## Best Practices -### Dla administratorów +### For administrators -1. **Włącz security w produkcji** - `--security-enabled` -2. **Ogranicz allowed_paths** - tylko niezbędne ścieżki -3. **Backup token store** - tokeny są nieodwracalne -4. **Rotacja tokenów** - okresowo revoke + create nowe +1. **Enable security in production** - `--security-enabled` +2. **Restrict allowed_paths** - only the necessary paths +3. **Back up the token store** - tokens are irreversible +4. **Rotate tokens** - periodically revoke + create new ones -### Dla agentów AI +### For AI agents -1. **Przechowuj tokeny bezpiecznie** - env vars lub secure storage -2. **Nie loguj tokenów** - unikaj wyświetlania w logach -3. **Używaj dedykowanych namespace'ów** - izolacja danych -4. **Sprawdzaj security_status** - upewnij się że security jest włączone +1. **Store tokens securely** - env vars or secure storage +2. **Do not log tokens** - avoid displaying them in logs +3. **Use dedicated namespaces** - data isolation +4. **Check security_status** - make sure security is enabled -## Przyszłe rozszerzenia +## Future Extensions -### Faza 3: Szyfrowanie Namespace'ów (planowane) +### Phase 3: Namespace Encryption (planned) ```toml -# Przyszła konfiguracja -[namespaces.pamietnik] +# Future configuration +[namespaces.diary] encrypted = true key_derivation = "argon2id" ``` -- Dane w LanceDB szyfrowane kluczem namespace'u -- Bez klucza = dane bezużyteczne -- Dla naprawdę wrażliwych danych +- Data in LanceDB encrypted with the namespace key +- Without the key = data is useless +- For truly sensitive data --- -Vibecrafted with AI Agents by Loctree (c)2025 The LibraxisAI Team -Co-Authored-By: [Maciej](void@div0.space) & [Klaudiusz](the1st@whoai.am) +Vibecrafted with AI Agents by Loctree (c)2025 Vetcoders diff --git a/docs/02_configuration.md b/docs/02_configuration.md index c1dfcfe..e06e2bb 100644 --- a/docs/02_configuration.md +++ b/docs/02_configuration.md @@ -1,49 +1,49 @@ # Configuration Guide -## Przegląd +## Overview -rust-memex można skonfigurować na trzy sposoby (w kolejności priorytetu): -1. **Flagi CLI** - najwyższy priorytet -2. **Plik konfiguracyjny TOML** - średni priorytet -3. **Wartości domyślne** - najniższy priorytet +rust-memex can be configured in three ways (in priority order): +1. **CLI flags** - highest priority +2. **TOML configuration file** - medium priority +3. **Default values** - lowest priority -## Opcje CLI +## CLI Options ```bash rust-memex [OPTIONS] [COMMAND] ``` -### Komendy +### Commands -| Komenda | Opis | +| Command | Description | |---------|------| -| `serve` | Uruchom serwer MCP (domyślna) | -| `wizard` | Interaktywny kreator konfiguracji | -| `index` | Batch indexing dokumentów | +| `serve` | Start the MCP server (default) | +| `wizard` | Interactive configuration wizard | +| `index` | Batch indexing of documents | -### Globalne Opcje +### Global Options -| Flaga | Opis | Domyślnie | +| Flag | Description | Default | |-------|------|-----------| -| `--config ` | Ścieżka do pliku konfiguracyjnego TOML | brak | -| `--cache-mb ` | Rozmiar cache w MB | `4096` | -| `--db-path ` | Ścieżka do LanceDB | `~/.rmcp-servers/rust-memex/lancedb` | -| `--max-request-bytes ` | Max rozmiar requestu | `5242880` (5MB) | -| `--log-level ` | Poziom logowania | `info` | -| `--allowed-paths ` | Dozwolone ścieżki (można powtórzyć) | `$HOME`, `cwd` | -| `--security-enabled` | Włącz namespace security | `false` | -| `--token-store-path ` | Ścieżka do token store | `~/.rmcp-servers/rust-memex/tokens.json` | +| `--config ` | Path to the TOML configuration file | none | +| `--cache-mb ` | Cache size in MB | `4096` | +| `--db-path ` | Path to LanceDB | `~/.rmcp-servers/rust-memex/lancedb` | +| `--max-request-bytes ` | Max request size | `5242880` (5MB) | +| `--log-level ` | Log level | `info` | +| `--allowed-paths ` | Allowed paths (can be repeated) | `$HOME`, `cwd` | +| `--security-enabled` | Enable namespace security | `false` | +| `--token-store-path ` | Path to the token store | `~/.rmcp-servers/rust-memex/tokens.json` | -### Przykłady CLI +### CLI Examples ```bash -# Podstawowe uruchomienie +# Basic run rust-memex serve -# Z własną konfiguracją +# With a custom configuration rust-memex serve --config ~/.rmcp-servers/rust-memex/config.toml -# Z security i custom paths +# With security and custom paths rust-memex serve \ --security-enabled \ --allowed-paths ~ \ @@ -54,71 +54,72 @@ rust-memex serve \ rust-memex index ./documents --namespace docs --recursive --glob "*.md" ``` -## Plik Konfiguracyjny (TOML) +## Configuration File (TOML) -### Lokalizacja +### Location -Domyślna lokalizacja: `~/.rmcp-servers/rust-memex/config.toml` +Default location: `~/.rmcp-servers/rust-memex/config.toml` -### Pełny Przykład +### Full Example ```toml -# Rozmiar cache w MB +# Cache size in MB cache_mb = 4096 -# Ścieżka do LanceDB vector store +# Path to the LanceDB vector store db_path = "~/.rmcp-servers/rust-memex/lancedb" -# Maksymalny rozmiar requestu JSON-RPC (bytes) +# Maximum JSON-RPC request size (bytes) max_request_bytes = 5242880 -# Poziom logowania: trace, debug, info, warn, error +# Log level: trace, debug, info, warn, error log_level = "info" -# Whitelist dozwolonych ścieżek dla operacji na plikach -# Jeśli puste, domyślnie $HOME i current working directory +# Whitelist of allowed paths for file operations +# If empty, defaults to $HOME and the current working directory allowed_paths = [ "~", - "/Volumes/LibraxisShare/Klaudiusz", + "/Volumes/Shared/notes", "/opt/shared/documents" ] -# Włącz namespace token-based access control +# Enable namespace token-based access control security_enabled = true -# Ścieżka do pliku z tokenami namespace'ów +# Path to the namespace tokens file token_store_path = "~/.rmcp-servers/rust-memex/tokens.json" ``` -### Minimalna Konfiguracja +### Minimal Configuration ```toml -# Tylko niezbędne ustawienia +# Only the essential settings db_path = "~/.rmcp-servers/rust-memex/lancedb" security_enabled = true ``` -## Tryby Serwera +## Server Modes -`rust-memex` udostępnia jeden kanoniczny MCP surface. Nie ma już osobnego -przełącznika `memory/full`, bo nie zmieniał on realnie kontraktu serwera. +`rust-memex` exposes a single canonical MCP surface. There is no longer a +separate `memory/full` switch, because it did not actually change the +server contract. -Jeśli chcesz zawęzić runtime: -- użyj `allowed_paths`, aby ograniczyć dostęp do filesystem -- ustaw `--security-enabled`, aby chronić namespace'y tokenami -- ustaw `--auth-token`, jeśli wystawiasz mutujące endpointy HTTP +If you want to narrow the runtime: +- use `allowed_paths` to restrict filesystem access +- set `--security-enabled` to protect namespaces with tokens +- set `--auth-token` if you expose mutating HTTP endpoints -## Zmienne Środowiskowe +## Environment Variables -| Zmienna | Opis | +| Variable | Description | |---------|------| -| `HOME` / `USERPROFILE` | Home directory (dla ~ expansion) | -| `LANCEDB_PATH` | Override ścieżki LanceDB | -| `SLED_PATH` | Override ścieżki sled K/V store | -| `FASTEMBED_CACHE_PATH` | Cache dla modeli FastEmbed | -| `HF_HUB_CACHE` | Cache dla modeli HuggingFace | +| `HOME` / `USERPROFILE` | Home directory (for ~ expansion) | +| `LANCEDB_PATH` | Override the LanceDB path | +| `SLED_PATH` | Override the sled K/V store path | +| `FASTEMBED_CACHE_PATH` | Cache for FastEmbed models | +| `HF_HUB_CACHE` | Cache for HuggingFace models | -## Konfiguracja dla Claude/MCP +## Configuration for Claude/MCP ### ~/.claude.json @@ -133,7 +134,7 @@ Jeśli chcesz zawęzić runtime: } ``` -### Z security enabled +### With security enabled ```json { @@ -153,88 +154,88 @@ Jeśli chcesz zawęzić runtime: ## Batch Indexing -Komenda `index` pozwala na masowe indeksowanie dokumentów. +The `index` command allows bulk indexing of documents. -### Składnia +### Syntax ```bash rust-memex index [OPTIONS] ``` -### Opcje +### Options -| Flaga | Opis | +| Flag | Description | |-------|------| -| `-n, --namespace ` | Namespace dla dokumentów (domyślnie: `rag`) | -| `-r, --recursive` | Rekursywnie przeglądaj podkatalogi | -| `-g, --glob ` | Filtruj pliki wzorcem glob | -| `--max-depth ` | Maksymalna głębokość (0 = bez limitu) | +| `-n, --namespace ` | Namespace for the documents (default: `rag`) | +| `-r, --recursive` | Traverse subdirectories recursively | +| `-g, --glob ` | Filter files by a glob pattern | +| `--max-depth ` | Maximum depth (0 = no limit) | -### Przykłady +### Examples ```bash -# Indeksuj pojedynczy plik +# Index a single file rust-memex index ./README.md -# Indeksuj folder rekursywnie +# Index a folder recursively rust-memex index ./docs --recursive --namespace documentation -# Tylko pliki markdown +# Only markdown files rust-memex index ./notes --recursive --glob "*.md" --namespace notes -# Z limitem głębokości +# With a depth limit rust-memex index ./project --recursive --max-depth 3 ``` -## Wizard (Kreator Konfiguracji) +## Wizard (Configuration Wizard) -Interaktywny kreator do generowania konfiguracji. +An interactive wizard for generating a configuration. ```bash rust-memex wizard -# Dry run - pokaż zmiany bez zapisywania +# Dry run - show changes without saving rust-memex wizard --dry-run ``` -Wizard pomoże skonfigurować: -- Ścieżkę do LanceDB -- Dozwolone ścieżki +The wizard helps you configure: +- The LanceDB path +- Allowed paths - Security settings -- Integrację z Claude +- Claude integration -## Priorytet Konfiguracji +## Configuration Priority -Gdy ta sama opcja jest ustawiona w wielu miejscach: +When the same option is set in multiple places: ``` CLI flag > Config file > Default value ``` -Przykład: +Example: ```bash # Config file: log_level = "info" # CLI: --log-level debug -# Wynik: debug (CLI wygrywa) +# Result: debug (CLI wins) rust-memex serve --config ~/.rmcp-servers/rust-memex/config.toml --log-level debug ``` -## Walidacja Konfiguracji +## Configuration Validation -Serwer waliduje konfigurację przy starcie: +The server validates the configuration at startup: -1. **Ścieżki** - sprawdza czy istnieją i są dostępne -2. **Allowed paths** - rozwiązuje ~ i sprawdza uprawnienia -3. **Token store** - tworzy plik jeśli nie istnieje (gdy security enabled) -4. **LanceDB** - inicjalizuje bazę jeśli nie istnieje +1. **Paths** - checks that they exist and are accessible +2. **Allowed paths** - resolves ~ and checks permissions +3. **Token store** - creates the file if it does not exist (when security enabled) +4. **LanceDB** - initializes the database if it does not exist -Błędy konfiguracji są raportowane przy starcie z jasnym komunikatem. +Configuration errors are reported at startup with a clear message. ## Troubleshooting ### "Access denied: path outside allowed directories" -Dodaj ścieżkę do `allowed_paths`: +Add the path to `allowed_paths`: ```toml allowed_paths = [ "~", @@ -244,19 +245,20 @@ allowed_paths = [ ### "Cannot resolve config path" -Sprawdź czy plik konfiguracyjny istnieje: +Check that the configuration file exists: ```bash ls -la ~/.rmcp-servers/rust-memex/config.toml ``` ### "Token store not found" -Przy pierwszym uruchomieniu z `--security-enabled`, token store jest tworzony automatycznie. Upewnij się że katalog nadrzędny istnieje: +On the first run with `--security-enabled`, the token store is created +automatically. Make sure the parent directory exists: ```bash mkdir -p ~/.rmcp-servers/rust-memex ``` -### Logi debugowania +### Debug logs ```bash rust-memex serve --log-level trace @@ -264,5 +266,4 @@ rust-memex serve --log-level trace --- -Vibecrafted with AI Agents by Loctree (c)2025 The LibraxisAI Team -Co-Authored-By: [Maciej](void@div0.space) & [Klaudiusz](the1st@whoai.am) +Vibecrafted with AI Agents by Loctree (c)2025 Vetcoders diff --git a/docs/ARCHITECTURE_ACTUAL.md b/docs/ARCHITECTURE_ACTUAL.md deleted file mode 100644 index b407988..0000000 --- a/docs/ARCHITECTURE_ACTUAL.md +++ /dev/null @@ -1,253 +0,0 @@ -# Memex Architecture - ACTUAL (Stan Faktyczny) - -```mermaid -flowchart TB - subgraph ENTRY["Entry Points"] - MCP["MCP Tools
(stdio JSON-RPC)"] - HTTP["HTTP API
(REST + SSE)"] - CLI["CLI
(commands)"] - end - - subgraph VALIDATION["Security Layer"] - PATH["Path Validation
• expand ~
• detect ..
• canonicalize
• whitelist check"] - TOKEN["Namespace Tokens
• create/revoke
• per-namespace auth"] - end - - subgraph EXTRACTION["Content Extraction"] - TXT["Plain Text (.txt)"] - PDF["PDF (.pdf)
pdf_extract"] - JSON_SMART["Smart JSON Detection
• Claude.ai export
• ChatGPT export
• Session essence
• Generic array"] - MD["Markdown (.md)
section extraction"] - CODE["Code (.rs/.py/.js)
❌ NO semantic chunking
treated as plain text"] - end - - subgraph DEDUP["Deduplication"] - HASH["SHA256 Hash
content_hash"] - CHECK["Storage Check
has_content_hash()"] - SKIP["Skip if EXISTS"] - end - - subgraph SLICING["Slicing Strategy"] - FLAT["FLAT MODE
512-char chunks
128-char overlap"] - ONION["ONION MODE (default)
4 hierarchical layers:
OUTER → MIDDLE → INNER → CORE"] - FAST["ONION-FAST
2 layers only
OUTER ↔ CORE"] - end - - subgraph EMBEDDING["Embedding Generation"] - MLX_ACTUAL["MLX Embedder
❌ Port: 12345 (not 8765)
Model: Qwen3-Embedding-8B-4bit-DWQ
Dims: 4096"] - OLLAMA["Ollama (fallback)
Port: 11434
qwen3-embedding:8b"] - BATCH["Smart Batching
max 64 items
max 128K chars"] - NO_VALIDATE["❌ NO Dimension Validation
Silent corruption possible"] - RETRY["Retry with Backoff
1s → 30s"] - end - - subgraph STORAGE["LanceDB Storage"] - RAMDISK["RAM Disk
/Volumes/MemexRAM
50GB HFS+"] - LANCE["LanceDB
~28GB vectors"] - SCHEMA["Schema v3
• id, namespace
• vector[4096]
• layer, parent_id
• content_hash"] - NO_ATOMIC["❌ NO Atomic Writes
Partial failures = ghost docs"] - end - - subgraph SEARCH["Search Pipeline"] - ROUTER["Query Router
auto-detect mode"] - VECTOR["Vector Search
ANN via IVF-HNSW"] - BM25["BM25 Full-Text"] - HYBRID["Hybrid Fusion
RRF scoring"] - NO_RERANK["❌ Reranker Optional
Falls back to cosine"] - end - - subgraph LAUNCHD["LaunchD Services"] - LD_RAMDISK["ai.libraxis.memex-ramdisk
Create 50GB RAM disk"] - LD_MLX_ACTUAL["ai.libraxis.mlx-embedding
❌ Port 12345 (not 8765)
WorkDir: vista-brain/scripts"] - LD_MEMEX["ai.libraxis.rust-memex
Port 8997 server
Uses RAM disk"] - LD_SNAPSHOT["ai.libraxis.memex-snapshot
Periodic sync"] - end - - subgraph E2E_TESTS["E2E Tests"] - NO_TEST_INDEX["❌ MISSING
test_index_search"] - NO_TEST_MCP["❌ MISSING
test_mcp_tools"] - NO_TEST_HTTP["❌ MISSING
test_http_api"] - NO_TEST_DEDUP["❌ MISSING
test_deduplication"] - end - - subgraph CONFIG["Configuration"] - NO_CONFIG["❌ NO config.toml
Uses env vars + hardcoded defaults"] - NO_MLX_FOLDER["❌ ~/.ai-memories/mlx-embeddings/
Contains unrelated project"] - end - - %% Flow connections - ENTRY --> VALIDATION - VALIDATION --> EXTRACTION - EXTRACTION --> DEDUP - DEDUP -->|NEW| SLICING - DEDUP -->|EXISTS| SKIP - SLICING --> EMBEDDING - EMBEDDING --> STORAGE - - %% Search flow - ENTRY --> SEARCH - SEARCH --> STORAGE - - %% LaunchD flow - LD_RAMDISK -->|creates| RAMDISK - LD_MLX_ACTUAL -->|provides| MLX_ACTUAL - LD_MEMEX -->|uses| RAMDISK - LD_MEMEX -.->|should call| MLX_ACTUAL - LD_SNAPSHOT -->|syncs| RAMDISK - - %% Styling - Green = works, Red = broken/missing, Yellow = partial - classDef works fill:#2d5a2d,stroke:#4a4,color:#fff - classDef broken fill:#5a2d2d,stroke:#a44,color:#fff - classDef partial fill:#5a5a2d,stroke:#aa4,color:#fff - classDef missing fill:#3d3d3d,stroke:#666,color:#888 - - class PATH,TOKEN,TXT,PDF,JSON_SMART,MD,HASH,CHECK,SKIP,FLAT,ONION,FAST,BATCH,RETRY,RAMDISK,LANCE,SCHEMA,ROUTER,VECTOR,BM25,HYBRID,LD_RAMDISK,LD_MEMEX,LD_SNAPSHOT works - class NO_VALIDATE,NO_ATOMIC,NO_RERANK,MLX_ACTUAL,LD_MLX_ACTUAL,NO_CONFIG,NO_MLX_FOLDER broken - class CODE,OLLAMA partial - class NO_TEST_INDEX,NO_TEST_MCP,NO_TEST_HTTP,NO_TEST_DEDUP missing -``` - -## Status: Co działa, co nie - -### ✅ DZIAŁA (zielone) -| Komponent | Status | Uwagi | -|-----------|--------|-------| -| Path Validation | ✅ | Pełna walidacja traversal | -| Namespace Tokens | ✅ | create/revoke/verify | -| Plain Text extraction | ✅ | UTF-8 read | -| PDF extraction | ✅ | pdf_extract crate | -| Smart JSON detection | ✅ | Claude/ChatGPT/Session | -| Markdown extraction | ✅ | Section-aware | -| Deduplication | ✅ | SHA256 + storage check | -| Flat/Onion/Fast slicing | ✅ | Wszystkie 3 tryby | -| Smart Batching | ✅ | 64 items / 128K chars | -| Retry with Backoff | ✅ | 1s → 30s | -| RAM Disk | ✅ | 50GB /Volumes/MemexRAM | -| LanceDB | ✅ | 28GB, schema v3 | -| Query Router | ✅ | auto-detect | -| Vector/BM25/Hybrid search | ✅ | Wszystkie tryby | -| LaunchD services | ✅ | ramdisk, memex, snapshot | - -### ❌ NIE DZIAŁA / NIE SPIĘTE (czerwone) -| Komponent | Problem | Impact | -|-----------|---------|--------| -| **MLX Port w kodzie** | Kod domyślnie 12345, serwer na **8765** | Config mismatch | -| **Atomic Writes** | BRAK - ghost docs przy crash | **HIGH** | -| **Reranker** | Optional, fallback to cosine | Słabsze wyniki | -| **config.toml** | BRAK - hardcoded defaults | Trudne zarządzanie | - -### ✅ NAPRAWIONE (w tej sesji) -| Komponent | Status | Uwagi | -|-----------|--------|-------| -| **Dimension Validation** | ✅ DODANE | `test_dimension()` w `EmbeddingClient::new()` | -| **E2E Tests** | ✅ DODANE | `tests/e2e_pipeline.rs` - 5 testów | -| **TextIntegrityMetrics** | ✅ DODANE | >90% threshold, audit command | -| **DimensionAdapter** | ✅ DODANE | Cross-dim search 1024/2048/4096 | -| **Audit/Purge commands** | ✅ DODANE | `rust-memex audit`, `purge-quality` | - -### ⚠️ CZĘŚCIOWE (żółte) -| Komponent | Status | Uwagi | -|-----------|--------|-------| -| Code extraction | ⚠️ | Traktowane jako plain text, brak AST | -| Ollama | ⚠️ | Działa jako fallback, ale to nie docelowy design | - -### ❌ CAŁKOWICIE BRAKUJE (szare) -| Komponent | Status | -|-----------|--------| -| E2E test: index → search | ❌ BRAK | -| E2E test: MCP tools | ❌ BRAK | -| E2E test: HTTP API | ❌ BRAK | -| E2E test: deduplication | ❌ BRAK | - ---- - -## Szczegóły LaunchD Services - -### Aktualny stan usług: -``` -PID STATUS SERVICE -10721 -15 ai.libraxis.rust-memex ← działa, port 8997 -46656 137 ai.libraxis.mlx-embedding ← działa, port 12345 (!) -46670 137 ai.libraxis.mlx-reranker ← działa -46743 1 ai.libraxis.mlx-batch-server -- 0 ai.libraxis.memex-snapshot -- 1 ai.libraxis.mlx-batch-runner -``` - -### ai.libraxis.mlx-embedding -``` -Port: 12345 (❌ powinien być 8765) -WorkDir: vista-brain/scripts/ (❌ powinien być ~/.ai-memories/mlx-embeddings/) -Script: mlx_embedding_server.py -Model: Qwen3-Embedding-8B-4bit-DWQ -Dims: 4096 ✅ -``` - -### ai.libraxis.rust-memex -``` -Port: 8997 ✅ -DB Path: /Volumes/MemexRAM/lancedb ✅ -Mode: --http-only ✅ -PathState: /Volumes/MemexRAM/lancedb (czeka na RAM disk) ✅ -``` - -### ai.libraxis.memex-ramdisk -``` -Size: 50GB (104857600 sectors) -Mount: /Volumes/MemexRAM ✅ -Source: ~/.ai-memories/lancedb -Sync: rsync -a ✅ -``` - ---- - -## Krytyczne luki do naprawy - -### 1. CRITICAL: Dimension Validation -```rust -// BRAK w embeddings/mod.rs -// Jeśli embedder zwróci 1024-dim: -// → Silent write to LanceDB -// → Cała baza corrupted -// → Brak recovery -``` - -### 2. HIGH: Atomic Batch Writes -```rust -// BRAK w storage/mod.rs -// Jeśli crash w połowie batch: -// → Ghost documents -// → Dedup nie złapie (inny hash?) -// → Brak rollback -``` - -### 3. MEDIUM: Port Mismatch -``` -OBIECANE: MLX embedder na 8765 -FAKTYCZNE: MLX embedder na 12345 - -rust-memex używa provider cascade: -1. Ollama localhost:11434 -2. Fallback dragon:12345 - -Więc działa, ale przez Ollama, nie przez dedykowany MLX! -``` - -### 4. MEDIUM: Missing Config -``` -OBIECANE: ~/.ai-memories/config.toml -FAKTYCZNE: Brak pliku - -Wszystko przez env vars lub hardcoded defaults. -Trudne do zarządzania na wielu maszynach. -``` - ---- - -## Propozycja naprawy (kolejność priorytetów) - -1. **[CRITICAL]** Dodać dimension validation w `EmbeddingClient::new()` -2. **[HIGH]** Implementować batch transaction z rollback -3. **[MEDIUM]** Zmienić port MLX na 8765 lub zaktualizować config memex -4. **[MEDIUM]** Stworzyć `~/.ai-memories/config.toml` z pełną konfiguracją -5. **[LOW]** Napisać testy E2E diff --git a/docs/ARCHITECTURE_COMPARISON.md b/docs/ARCHITECTURE_COMPARISON.md deleted file mode 100644 index 1e34397..0000000 --- a/docs/ARCHITECTURE_COMPARISON.md +++ /dev/null @@ -1,230 +0,0 @@ -# Memex Architecture Comparison: PROMISED vs ACTUAL - -## Quick Status - -``` -╔═══════════════════════════════════════════════════════════════════════════╗ -║ OVERALL STATUS: ~85% Complete ║ -╠═══════════════════════════════════════════════════════════════════════════╣ -║ ✅ Working: 20 components ║ -║ ✅ Fixed: 5 components (in this session) ║ -║ ❌ Broken: 2 components (1 HIGH) ║ -║ ⚠️ Partial: 2 components ║ -║ ⚠️ Config: Port mismatch (works via Ollama fallback) ║ -╚═══════════════════════════════════════════════════════════════════════════╝ -``` - ---- - -## Side-by-Side Comparison - -| Component | PROMISED | ACTUAL | Status | -|-----------|----------|--------|--------| -| **Entry Points** | -| MCP Tools | JSON-RPC over stdio | JSON-RPC over stdio | ✅ | -| HTTP API | REST + SSE on 8997 | REST + SSE on 8997 | ✅ | -| CLI | index/search/optimize | index/search/optimize | ✅ | -| **Security** | -| Path Validation | expand ~ / detect .. / canonicalize | expand ~ / detect .. / canonicalize | ✅ | -| Namespace Tokens | create/revoke/verify | create/revoke/verify | ✅ | -| **Extraction** | -| Plain Text | UTF-8 read | UTF-8 read | ✅ | -| PDF | pdf_extract crate | pdf_extract crate | ✅ | -| JSON (smart) | Claude/ChatGPT/Session detection | Claude/ChatGPT/Session detection | ✅ | -| Markdown | Section extraction | Section extraction | ✅ | -| Code | Semantic chunking (AST) | Plain text only | ⚠️ Partial | -| **Deduplication** | -| Hash Algorithm | SHA256 | SHA256 | ✅ | -| Storage Check | has_content_hash() | has_content_hash() | ✅ | -| **Slicing** | -| Flat Mode | 512-char / 128 overlap | 512-char / 128 overlap | ✅ | -| Onion Mode | 4 layers (outer→core) | 4 layers (outer→core) | ✅ | -| Onion-Fast | 2 layers | 2 layers | ✅ | -| **Embedding** | -| Embedder Port | **8765** | **12345** | ❌ Mismatch | -| Embedder Location | `~/.ai-memories/mlx-embeddings/` | `vista-brain/scripts/` | ❌ Wrong | -| Model | Qwen3-Embedding-8B | Qwen3-Embedding-8B-4bit-DWQ | ✅ | -| Dimensions | 4096 | 4096 | ✅ | -| **Dimension Validation** | **Fail fast on mismatch** | **✅ test_dimension() in EmbeddingClient::new()** | ✅ FIXED | -| **DimensionAdapter** | Cross-dim 1024/2048/4096 | **✅ expand/contract adapters** | ✅ FIXED | -| Batching | 64 items / 128K chars | 64 items / 128K chars | ✅ | -| Retry | Exponential backoff | Exponential backoff | ✅ | -| Fallback | Dragon remote | Ollama (unintended) | ⚠️ Partial | -| **Storage** | -| RAM Disk | 50GB /Volumes/MemexRAM | 50GB /Volumes/MemexRAM | ✅ | -| LanceDB Size | ~28GB | ~28GB | ✅ | -| Schema | v3 with content_hash | v3 with content_hash | ✅ | -| **Atomic Writes** | **Transaction rollback** | **NONE - ghost docs** | ❌ HIGH | -| **Search** | -| Query Router | auto-detect | auto-detect | ✅ | -| Vector Search | ANN via IVF-HNSW | ANN via IVF-HNSW | ✅ | -| BM25 | Full-text | Full-text | ✅ | -| Hybrid | RRF fusion | RRF fusion | ✅ | -| Reranker | Dedicated on 8766 | Optional (cosine fallback) | ❌ Weak | -| **LaunchD** | -| memex-ramdisk | Create 50GB | Create 50GB | ✅ | -| mlx-embedding | Port 8765 | Port 12345 | ❌ Mismatch | -| rust-memex | Port 8997, uses RAM | Port 8997, uses RAM | ✅ | -| memex-snapshot | Periodic sync | Periodic sync | ✅ | -| **Config** | -| Config File | `~/.ai-memories/config.toml` | **NONE** | ❌ Missing | -| **Quality Assurance** | -| TextIntegrityMetrics | >90% threshold | **✅ compute() + recommendation()** | ✅ FIXED | -| Audit command | Per-namespace check | **✅ rust-memex audit** | ✅ FIXED | -| Purge command | Remove low-quality | **✅ rust-memex purge-quality** | ✅ FIXED | -| **Testing** | -| E2E: pipeline | Required | **✅ tests/e2e_pipeline.rs (5 tests)** | ✅ FIXED | -| E2E: MCP tools | Required | **MISSING** | ❌ | -| E2E: HTTP API | Required | **MISSING** | ❌ | -| Unit tests | Required | ~20 tests | ✅ | - ---- - -## Visual Comparison - -### PROMISED Architecture -``` -┌─────────────────────────────────────────────────────────────────┐ -│ ENTRY POINTS │ -│ MCP (stdio) │ HTTP (8997) │ CLI │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SECURITY + EXTRACTION │ -│ Path Validation │ Token Auth │ PDF/JSON/MD/Code extraction │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ DEDUPLICATION │ -│ SHA256 → check storage → skip if exists │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SLICING (Flat/Onion/Fast) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ MLX EMBEDDER (port 8765) │ -│ ~/.ai-memories/mlx-embeddings/ │ Qwen3-Embedding-8B │ 4096d │ -│ ✓ DIMENSION VALIDATION (fail fast) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ LANCEDB (RAM DISK) │ -│ /Volumes/MemexRAM │ 50GB │ ~28GB vectors │ schema v3 │ -│ ✓ ATOMIC BATCH WRITES (transaction) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SEARCH │ -│ Router → Vector/BM25/Hybrid → Reranker (8766) → Results │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ E2E TESTS │ -│ index→search │ MCP tools │ HTTP API │ deduplication │ -└─────────────────────────────────────────────────────────────────┘ -``` - -### ACTUAL Architecture (Updated) -``` -┌─────────────────────────────────────────────────────────────────┐ -│ ENTRY POINTS │ -│ MCP (stdio) │ HTTP (8997) │ CLI (rust-memex) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SECURITY + EXTRACTION │ -│ Path Validation │ Token Auth │ PDF/JSON/MD │ ⚠️Code=text │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ DEDUPLICATION │ -│ SHA256 → check storage → skip if exists │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SLICING (Flat/Onion/Fast) │ -│ ✅ TextIntegrityMetrics (>90% threshold) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ ⚠️ MLX EMBEDDER (port 12345, not 8765) │ -│ vista-brain/scripts/ │ Qwen3-Embedding-8B-4bit │ 4096d │ -│ ✅ DIMENSION VALIDATION (test_dimension()) │ -│ ✅ DimensionAdapter (cross-dim 1024/2048/4096) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ LANCEDB (RAM DISK) │ -│ /Volumes/MemexRAM │ 50GB │ ~28GB vectors │ schema v3 │ -│ ❌ NO ATOMIC WRITES (ghost docs on crash) │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ SEARCH │ -│ Router → Vector/BM25/Hybrid → ⚠️ cosine fallback → Results │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ ✅ QUALITY ASSURANCE │ -│ ✅ E2E pipeline │ ✅ audit cmd │ ✅ purge cmd │ ❌ HTTP tests │ -└─────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Priority Fix List - -### ✅ FIXED (This Session) -1. ~~**Dimension Validation**~~ → ✅ `test_dimension()` in `EmbeddingClient::new()` -2. ~~**E2E Tests**~~ → ✅ `tests/e2e_pipeline.rs` (5 tests) -3. ~~**Quality Metrics**~~ → ✅ `TextIntegrityMetrics` with >90% threshold -4. ~~**Cross-dim Search**~~ → ✅ `DimensionAdapter` (1024/2048/4096) -5. ~~**Audit/Purge**~~ → ✅ `rust-memex audit` + `purge-quality` commands - -### 🟠 HIGH (Data Integrity Risk) -1. **Atomic Batch Writes** - Implement transaction wrapper - ```rust - // Wrap batch operations in transaction - storage.begin_transaction()?; - for doc in batch { - storage.add(doc)?; - } - storage.commit()?; // or rollback on error - ``` - -### 🟡 MEDIUM (Config/Port Mismatch) -2. **MLX Port** - Change launch agent to port 8765 OR update memex config -3. **Config File** - Create `~/.ai-memories/config.toml` with all settings - -### 🟢 LOW (Nice to Have) -4. **Code Semantic Chunking** - Add AST-based chunking for .rs/.py/.js -5. **Reranker Integration** - Make reranker non-optional -6. **HTTP API Tests** - Add E2E tests for REST endpoints - ---- - -## Files Created - -1. `docs/ARCHITECTURE_PROMISED.md` - Docelowa architektura (Mermaid) -2. `docs/ARCHITECTURE_ACTUAL.md` - Stan faktyczny (Mermaid + czerwone krzyże) -3. `docs/ARCHITECTURE_COMPARISON.md` - Porównanie side-by-side (ten plik) - ---- - -*Vibecrafted with AI Agents by Vetcoders (c)2024-2026 Loctree* diff --git a/docs/ARCHITECTURE_PROMISED.md b/docs/ARCHITECTURE_PROMISED.md deleted file mode 100644 index 9b48eeb..0000000 --- a/docs/ARCHITECTURE_PROMISED.md +++ /dev/null @@ -1,127 +0,0 @@ -# Memex Architecture - PROMISED (Docelowa) - -```mermaid -flowchart TB - subgraph ENTRY["Entry Points"] - MCP["MCP Tools
(stdio JSON-RPC)"] - HTTP["HTTP API
(REST + SSE)"] - CLI["CLI
(commands)"] - end - - subgraph VALIDATION["Security Layer"] - PATH["Path Validation
• expand ~
• detect ..
• canonicalize
• whitelist check"] - TOKEN["Namespace Tokens
• create/revoke
• per-namespace auth"] - end - - subgraph EXTRACTION["Content Extraction"] - TXT["Plain Text (.txt)"] - PDF["PDF (.pdf)
pdf_extract"] - JSON_SMART["Smart JSON Detection
• Claude.ai export
• ChatGPT export
• Session essence
• Generic array"] - MD["Markdown (.md)
section extraction"] - CODE["Code (.rs/.py/.js)
semantic chunking"] - end - - subgraph DEDUP["Deduplication"] - HASH["SHA256 Hash
content_hash"] - CHECK["Storage Check
has_content_hash()"] - SKIP["Skip if EXISTS"] - end - - subgraph SLICING["Slicing Strategy"] - FLAT["FLAT MODE
512-char chunks
128-char overlap"] - ONION["ONION MODE (default)
4 hierarchical layers:
OUTER → MIDDLE → INNER → CORE"] - FAST["ONION-FAST
2 layers only
OUTER ↔ CORE"] - end - - subgraph EMBEDDING["Embedding Generation"] - MLX["MLX Native Embedder
Port: 8765
Model: Qwen3-Embedding-8B
Dims: 4096"] - BATCH["Smart Batching
max 64 items
max 128K chars"] - VALIDATE["Dimension Validation
FAIL FAST if mismatch"] - RETRY["Retry with Backoff
1s → 30s"] - end - - subgraph STORAGE["LanceDB Storage"] - RAMDISK["RAM Disk
/Volumes/MemexRAM
50GB HFS+"] - LANCE["LanceDB
~28GB vectors"] - SCHEMA["Schema v3
• id, namespace
• vector[4096]
• layer, parent_id
• content_hash"] - ATOMIC["Atomic Batch Writes
Transaction rollback"] - end - - subgraph SEARCH["Search Pipeline"] - ROUTER["Query Router
auto-detect mode"] - VECTOR["Vector Search
ANN via IVF-HNSW"] - BM25["BM25 Full-Text"] - HYBRID["Hybrid Fusion
RRF scoring"] - RERANK["Reranker
Port: 8766
cross-encoder"] - end - - subgraph LAUNCHD["LaunchD Services"] - LD_RAMDISK["ai.libraxis.memex-ramdisk
Create 50GB RAM disk"] - LD_MLX["ai.libraxis.mlx-embedding
Port 8765 embedder"] - LD_MEMEX["ai.libraxis.rust-memex
Port 8997 server"] - LD_SNAPSHOT["ai.libraxis.memex-snapshot
Periodic sync to disk"] - end - - subgraph E2E_TESTS["E2E Tests"] - TEST_INDEX["test_index_search
File → Vector → Search"] - TEST_MCP["test_mcp_tools
JSON-RPC handlers"] - TEST_HTTP["test_http_api
REST endpoints"] - TEST_DEDUP["test_deduplication
Hash collision handling"] - end - - %% Flow connections - ENTRY --> VALIDATION - VALIDATION --> EXTRACTION - EXTRACTION --> DEDUP - DEDUP -->|NEW| SLICING - DEDUP -->|EXISTS| SKIP - SLICING --> EMBEDDING - EMBEDDING --> STORAGE - - %% Search flow - ENTRY --> SEARCH - SEARCH --> STORAGE - - %% LaunchD flow - LD_RAMDISK -->|creates| RAMDISK - LD_MLX -->|provides| MLX - LD_MEMEX -->|uses| RAMDISK - LD_MEMEX -->|calls| MLX - LD_SNAPSHOT -->|syncs| RAMDISK - - %% Styling - classDef promised fill:#2d5a2d,stroke:#4a4,color:#fff - classDef critical fill:#5a2d2d,stroke:#a44,color:#fff - - class MLX,VALIDATE,ATOMIC,TEST_INDEX,TEST_MCP,TEST_HTTP,TEST_DEDUP,LD_MLX promised - class RAMDISK,LANCE,SCHEMA critical -``` - -## Kluczowe założenia docelowej architektury - -### 1. Embedding Layer -- **Dedykowany MLX embedder** na porcie **8765** -- Folder implementacji: `~/.ai-memories/mlx-embeddings/` -- Model: `Qwen3-Embedding-8B` (4096 dims) -- **NIE Ollama** - natywny MLX dla Dragona - -### 2. RAM Disk Architecture -- 50GB RAM disk `/Volumes/MemexRAM` -- LanceDB (~28GB) całkowicie w RAM -- Snapshot daemon - periodic sync to `~/.ai-memories/lancedb` -- PathState dependency - memex startuje dopiero gdy RAM disk gotowy - -### 3. Dimension Safety -- **Validation at startup** - sprawdzenie czy embedder zwraca 4096 dims -- **Fail fast** - crash jeśli mismatch, nie silent corruption - -### 4. Atomic Writes -- Transaction boundaries na batch writes -- Rollback przy partial failures -- No ghost documents - -### 5. E2E Test Coverage -- Full pipeline tests (index → embed → store → search → verify) -- MCP tool integration tests -- HTTP API endpoint tests -- Deduplication regression tests diff --git a/docs/HTTP_API.md b/docs/HTTP_API.md index 991d48e..d75a631 100644 --- a/docs/HTTP_API.md +++ b/docs/HTTP_API.md @@ -56,19 +56,19 @@ Example response: "status": "ok", "hint": "OK", "version": "0.5.1", - "db_path": "/Users/you/.rmcp-servers/rust-memex/lancedb", + "db_path": "/Users/tester/.rmcp-servers/rust-memex/lancedb", "embedding_provider": "ollama-local", - "total_documents": 27455, + "total_documents": 20000, "namespace_count": 2, "namespaces": [ { - "id": "kodowanie", - "count": 15000, + "id": "notes", + "count": 12000, "last_indexed_at": "2026-03-15T10:11:12Z" }, { "id": "memories", - "count": 12455, + "count": 8000, "last_indexed_at": null } ] @@ -129,7 +129,7 @@ These remain useful for legacy clients, but new code should prefer `GET /api/dis ```bash curl -X POST http://localhost:8997/search \ -H "Content-Type: application/json" \ - -d '{"query":"rust async","namespace":"kodowanie","k":10,"mode":"hybrid","project":"Vista","deep":true}' + -d '{"query":"rust async","namespace":"notes","k":10,"mode":"hybrid","project":"Demo","deep":true}' ``` Representative response: @@ -139,7 +139,7 @@ Representative response: "results": [ { "id": "doc-456", - "namespace": "kodowanie", + "namespace": "notes", "text": "Matching content...", "score": 0.85, "layer": "L0_Atom" diff --git a/docs/INDEXING_GUIDE.md b/docs/INDEXING_GUIDE.md index e4c749e..ddf06db 100644 --- a/docs/INDEXING_GUIDE.md +++ b/docs/INDEXING_GUIDE.md @@ -93,4 +93,4 @@ To normalne dla dużych plików. 8MB JSON = ~15-20 minut z Qwen3-Embedding-8B. --- -Vibecrafted with AI Agents by Loctree (c)2025 The LibraxisAI Team +Vibecrafted with AI Agents by Vetcoders (c)2025 diff --git a/scripts/install.sh b/scripts/install.sh index 196f980..d00b959 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -37,5 +37,5 @@ JSON echo "" echo "Notes:" -echo "- The MLX HTTP bridge is optional. By default, DRAGON_BASE_URL=http://localhost." +echo "- The MLX HTTP bridge is optional. By default, EMBEDDER_BASE_URL=http://localhost." echo "- To force local-only embeddings, set DISABLE_MLX=1." diff --git a/scripts/reindex-memories.sh b/scripts/reindex-memories.sh index ad7e10f..57d8999 100755 --- a/scripts/reindex-memories.sh +++ b/scripts/reindex-memories.sh @@ -2,7 +2,7 @@ # reindex-memories.sh # Reindex all memories after fixing timestamp preservation (P0) # -# Created by M&K (c)2025 The LibraxisAI Team +# Created by Vetcoders (c)2025 # Part of rust-memex P4 fix set -e diff --git a/src/bin/cli/config.rs b/src/bin/cli/config.rs index c26723d..bec794b 100644 --- a/src/bin/cli/config.rs +++ b/src/bin/cli/config.rs @@ -157,8 +157,8 @@ pub struct MlxFileConfig { #[serde(default)] pub disabled: bool, pub local_port: Option, - pub dragon_url: Option, - pub dragon_port: Option, + pub embedder_url: Option, + pub embedder_port: Option, pub embedder_model: Option, pub reranker_model: Option, pub reranker_port_offset: Option, @@ -171,8 +171,8 @@ impl MlxFileConfig { config.merge_file_config(rust_memex::MlxMergeOptions { disabled: Some(self.disabled), local_port: self.local_port, - dragon_url: self.dragon_url.clone(), - dragon_port: self.dragon_port, + embedder_url: self.embedder_url.clone(), + embedder_port: self.embedder_port, embedder_model: self.embedder_model.clone(), reranker_model: self.reranker_model.clone(), reranker_port_offset: self.reranker_port_offset, diff --git a/src/bin/cli/data.rs b/src/bin/cli/data.rs index 2012390..f065250 100644 --- a/src/bin/cli/data.rs +++ b/src/bin/cli/data.rs @@ -1180,7 +1180,7 @@ mod tests { metadata: json!({ "original_id": "doc-1", "layer": "outer", - "project": "vista" + "project": "demo" }), content_hash: Some("outer-hash".to_string()), embeddings: None, @@ -1192,7 +1192,7 @@ mod tests { metadata: json!({ "original_id": "doc-1", "layer": "core", - "project": "vista" + "project": "demo" }), content_hash: Some("core-hash".to_string()), embeddings: None, @@ -1239,7 +1239,7 @@ mod tests { "layer": "outer", "slice_mode": "onion", "content_hash": "stale", - "project": "vista" + "project": "demo" }); let prepared = prepare_reprocess_metadata( @@ -1252,7 +1252,7 @@ mod tests { true, ); - assert_eq!(prepared["project"], "vista"); + assert_eq!(prepared["project"], "demo"); assert_eq!(prepared["slice_mode"], "onion-fast"); assert_eq!(prepared["reprocess_source_record_id"], "core-1"); assert_eq!(prepared["reprocess_source_hash"], "fresh-hash"); diff --git a/src/bin/cli/definition.rs b/src/bin/cli/definition.rs index e8ee8a9..124f85e 100644 --- a/src/bin/cli/definition.rs +++ b/src/bin/cli/definition.rs @@ -202,8 +202,8 @@ pub enum Commands { /// full metadata, and related chunks. /// /// Examples: - /// rust-memex dive -n memories -q "dragon" - /// rust-memex dive -n memories -q "dragon" --verbose + /// rust-memex dive -n memories -q "laptop" + /// rust-memex dive -n memories -q "laptop" --verbose Dive { /// Namespace to search in #[arg(long, short = 'n', required = true)] @@ -316,10 +316,10 @@ pub enum Commands { /// defaults. Results include relevance scores, timestamps, and metadata. /// /// Examples: - /// rust-memex search -n memories -q "when did we buy dragon" - /// rust-memex search -n memories -q "dragon" --deep - /// rust-memex search -n memories -q "dragon" -l 20 - /// rust-memex search -n memories -q "dragon" --mode hybrid + /// rust-memex search -n memories -q "when did we buy laptop" + /// rust-memex search -n memories -q "laptop" --deep + /// rust-memex search -n memories -q "laptop" -l 20 + /// rust-memex search -n memories -q "laptop" --mode hybrid Search { /// Namespace to search in #[arg(long, short = 'n', required = true)] @@ -497,8 +497,8 @@ pub enum Commands { /// using the onion slice architecture (outer layers = summaries). /// /// Examples: - /// rust-memex recall "Vista architecture" # Search all namespaces - /// rust-memex recall "dragon setup" -n memories # Specific namespace + /// rust-memex recall "Demo architecture" # Search all namespaces + /// rust-memex recall "laptop setup" -n memories # Specific namespace /// rust-memex recall "auth flow" --limit 20 # More sources Recall { /// What to recall (search query) @@ -665,7 +665,7 @@ pub enum Commands { /// Examples: /// rust-memex merge --source ~/db1 --source ~/db2 --target ~/merged /// rust-memex merge --source ~/db1 --source ~/db2 --target ~/merged --dedup - /// rust-memex merge --source ~/dragon-db --target ~/merged --namespace-prefix "dragon:" + /// rust-memex merge --source ~/laptop-db --target ~/merged --namespace-prefix "laptop:" /// rust-memex merge --source ~/db1 --target ~/merged --dry-run Merge { /// Source database paths (can specify multiple times) @@ -680,7 +680,7 @@ pub enum Commands { #[arg(long, short = 'd')] dedup: bool, - /// Prefix to add to source namespaces (e.g., "dragon:" -> "dragon:memories") + /// Prefix to add to source namespaces (e.g., "laptop:" -> "laptop:memories") #[arg(long, short = 'p')] namespace_prefix: Option, diff --git a/src/bin/cli/inspection.rs b/src/bin/cli/inspection.rs index 0b96ac2..03f68ff 100644 --- a/src/bin/cli/inspection.rs +++ b/src/bin/cli/inspection.rs @@ -154,7 +154,7 @@ pub async fn run_overview( } } let mut top_keywords: Vec<_> = keyword_counts.into_iter().collect(); - top_keywords.sort_by(|a, b| b.1.cmp(&a.1)); + top_keywords.sort_by_key(|b| std::cmp::Reverse(b.1)); let top_keywords: Vec<(String, usize)> = top_keywords.into_iter().take(10).collect(); // Check for timestamps in metadata (look for common timestamp patterns) diff --git a/src/embeddings/mod.rs b/src/embeddings/mod.rs index 3f4943c..d8afb2c 100644 --- a/src/embeddings/mod.rs +++ b/src/embeddings/mod.rs @@ -17,8 +17,8 @@ //! priority = 1 //! //! [[embeddings.providers]] -//! name = "dragon" -//! base_url = "http://dragon:12345" +//! name = "embedder" +//! base_url = "http://localhost:12345" //! model = "Qwen/Qwen3-Embedding-4B" //! priority = 2 //! ``` @@ -174,8 +174,8 @@ impl Default for EmbeddingConfig { endpoint: default_embeddings_endpoint(), }, ProviderConfig { - name: "dragon".to_string(), - base_url: "http://dragon:12345".to_string(), + name: "embedder".to_string(), + base_url: "http://localhost:12345".to_string(), model: "Qwen/Qwen3-Embedding-4B".to_string(), priority: 2, endpoint: default_embeddings_endpoint(), @@ -218,8 +218,8 @@ impl EmbeddingConfig { pub struct MlxConfig { pub disabled: bool, pub local_port: u16, - pub dragon_url: String, - pub dragon_port: u16, + pub embedder_url: String, + pub embedder_port: u16, pub embedder_model: String, pub reranker_model: String, pub reranker_port_offset: u16, @@ -232,8 +232,8 @@ pub struct MlxConfig { pub struct MlxMergeOptions { pub disabled: Option, pub local_port: Option, - pub dragon_url: Option, - pub dragon_port: Option, + pub embedder_url: Option, + pub embedder_port: Option, pub embedder_model: Option, pub reranker_model: Option, pub reranker_port_offset: Option, @@ -244,8 +244,8 @@ impl Default for MlxConfig { Self { disabled: false, local_port: 12345, - dragon_url: "http://dragon".to_string(), - dragon_port: 12345, + embedder_url: "http://localhost".to_string(), + embedder_port: 12345, embedder_model: "Qwen/Qwen3-Embedding-4B".to_string(), reranker_model: "Qwen/Qwen3-Reranker-4B".to_string(), reranker_port_offset: 1, @@ -267,10 +267,10 @@ impl MlxConfig { .and_then(|s| s.parse().ok()) .unwrap_or(12345); - let dragon_url = - std::env::var("DRAGON_BASE_URL").unwrap_or_else(|_| "http://dragon".to_string()); + let embedder_url = + std::env::var("EMBEDDER_BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()); - let dragon_port = std::env::var("DRAGON_EMBEDDER_PORT") + let embedder_port = std::env::var("EMBEDDER_PORT") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(local_port); @@ -300,8 +300,8 @@ impl MlxConfig { Self { disabled, local_port, - dragon_url, - dragon_port, + embedder_url, + embedder_port, embedder_model, reranker_model, reranker_port_offset, @@ -318,11 +318,11 @@ impl MlxConfig { if let Some(v) = opts.local_port { self.local_port = v; } - if let Some(v) = opts.dragon_url { - self.dragon_url = v; + if let Some(v) = opts.embedder_url { + self.embedder_url = v; } - if let Some(v) = opts.dragon_port { - self.dragon_port = v; + if let Some(v) = opts.embedder_port { + self.embedder_port = v; } if let Some(v) = opts.embedder_model { self.embedder_model = v; @@ -353,15 +353,15 @@ impl MlxConfig { endpoint: default_embeddings_endpoint(), }, ProviderConfig { - name: "dragon".to_string(), - base_url: format!("{}:{}", self.dragon_url, self.dragon_port), + name: "embedder".to_string(), + base_url: format!("{}:{}", self.embedder_url, self.embedder_port), model: self.embedder_model.clone(), priority: 2, endpoint: default_embeddings_endpoint(), }, ], reranker: RerankerConfig { - base_url: Some(format!("{}:{}", self.dragon_url, reranker_port)), + base_url: Some(format!("{}:{}", self.embedder_url, reranker_port)), model: Some(self.reranker_model.clone()), endpoint: default_rerank_endpoint(), }, @@ -1344,8 +1344,8 @@ mod tests { let legacy = MlxConfig { disabled: false, local_port: 12345, - dragon_url: "http://dragon".into(), - dragon_port: 12345, + embedder_url: "http://localhost".into(), + embedder_port: 12345, embedder_model: "test-model".into(), reranker_model: "rerank-model".into(), reranker_port_offset: 1, diff --git a/src/engine.rs b/src/engine.rs index 83313e1..8d7812a 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -398,7 +398,7 @@ impl LayerStats { // Sort by frequency and take top 10 let mut keywords: Vec<_> = keyword_counts.into_iter().collect(); - keywords.sort_by(|a, b| b.1.cmp(&a.1)); + keywords.sort_by_key(|b| std::cmp::Reverse(b.1)); let top_keywords = keywords.into_iter().take(10).map(|(k, _)| k).collect(); Self { @@ -506,34 +506,34 @@ impl MemexEngine { /// # Example /// /// ```rust,ignore - /// let engine = MemexEngine::for_app("vista", "patient-notes").await?; + /// let engine = MemexEngine::for_app("myapp", "notes").await?; /// ``` pub async fn for_app(app_name: &str, namespace: &str) -> Result { let config = MemexConfig::new(app_name, namespace); Self::new(config).await } - /// Vista-optimized setup with 1024-dimension embeddings. + /// Optimized setup with 1024-dimension embeddings. /// /// Uses smaller embedding model (qwen3-embedding:0.6b) for faster inference. /// /// # Example /// /// ```rust,ignore - /// let engine = MemexEngine::for_vista().await?; + /// let engine = MemexEngine::for_app_optimized().await?; /// ``` - pub async fn for_vista() -> Result { + pub async fn for_app_optimized() -> Result { use crate::embeddings::ProviderConfig; let config = MemexConfig { - app_name: "vista".to_string(), + app_name: "app".to_string(), namespace: "default".to_string(), - db_path: Some("~/.rmcp-servers/vista/lancedb".to_string()), + db_path: Some("~/.rmcp-servers/app/lancedb".to_string()), dimension: 1024, embedding_config: EmbeddingConfig { required_dimension: 1024, providers: vec![ProviderConfig { - name: "ollama-vista".to_string(), + name: "ollama-optimized".to_string(), base_url: "http://localhost:11434".to_string(), model: "qwen3-embedding:0.6b".to_string(), priority: 1, @@ -543,7 +543,7 @@ impl MemexEngine { }, enable_bm25: false, bm25_config: None, - enable_hybrid: true, // Hybrid enabled for Vista + enable_hybrid: true, // Hybrid enabled for the optimized profile hybrid_config: None, }; Self::new(config).await @@ -676,7 +676,7 @@ impl MemexEngine { /// # Example /// /// ```rust,ignore - /// let results = engine.search_hybrid("when did we buy dragon", 10).await?; + /// let results = engine.search_hybrid("when did we buy laptop", 10).await?; /// for r in results { /// println!("{}: combined={:.3}, vector={:?}, bm25={:?}", /// r.id, r.combined_score, r.vector_score, r.bm25_score); @@ -721,7 +721,7 @@ impl MemexEngine { /// use rust_memex::SearchMode; /// /// // Keyword-only for exact matches - /// let results = engine.search_with_mode("dragon", 10, SearchMode::Keyword).await?; + /// let results = engine.search_with_mode("laptop", 10, SearchMode::Keyword).await?; /// ``` pub async fn search_with_mode( &self, @@ -853,7 +853,7 @@ impl MemexEngine { let mut docs = Vec::with_capacity(items.len()); let mut bm25_docs = Vec::new(); - for (item, embedding) in items.iter().zip(embeddings.into_iter()) { + for (item, embedding) in items.iter().zip(embeddings) { let doc = ChromaDocument::new_flat( item.id.clone(), self.namespace.clone(), @@ -1139,12 +1139,12 @@ mod tests { #[test] fn test_memex_config_builder() { - let config = MemexConfig::new("vista", "patients") + let config = MemexConfig::new("demo", "records") .with_dimension(1024) .with_db_path("/custom/path/db"); - assert_eq!(config.app_name, "vista"); - assert_eq!(config.namespace, "patients"); + assert_eq!(config.app_name, "demo"); + assert_eq!(config.namespace, "records"); assert_eq!(config.dimension, 1024); assert_eq!(config.embedding_config.required_dimension, 1024); assert_eq!(config.effective_db_path(), "/custom/path/db"); diff --git a/src/http/mod.rs b/src/http/mod.rs index 0352a8b..47b9478 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -2609,20 +2609,20 @@ mod tests { #[test] fn test_search_request_accepts_k_alias() { - let json = r#"{"query": "test", "k": 7, "deep": true, "project": "Vista"}"#; + let json = r#"{"query": "test", "k": 7, "deep": true, "project": "Demo"}"#; let req: SearchRequest = serde_json::from_str(json).unwrap(); assert_eq!(req.limit, 7); assert!(req.deep); - assert_eq!(req.project.as_deref(), Some("Vista")); + assert_eq!(req.project.as_deref(), Some("Demo")); } #[test] fn test_sse_search_params_accept_k_alias() { - let json = r#"{"query":"test","k":9,"deep":true,"project":"Vista","mode":"bm25"}"#; + let json = r#"{"query":"test","k":9,"deep":true,"project":"Demo","mode":"bm25"}"#; let params: SseSearchParams = serde_json::from_str(json).unwrap(); assert_eq!(params.limit, 9); assert!(params.deep); - assert_eq!(params.project.as_deref(), Some("Vista")); + assert_eq!(params.project.as_deref(), Some("Demo")); assert_eq!(params.mode, "bm25"); } diff --git a/src/mcp_protocol.rs b/src/mcp_protocol.rs index 3925b47..f7559f6 100644 --- a/src/mcp_protocol.rs +++ b/src/mcp_protocol.rs @@ -1164,10 +1164,10 @@ mod tests { #[test] fn requested_search_options_captures_project_filter() { assert_eq!( - requested_search_options(&json!({"project": "Vista"})), + requested_search_options(&json!({"project": "Demo"})), SearchOptions { layer_filter: Some(SliceLayer::Outer), - project_filter: Some("Vista".to_string()), + project_filter: Some("Demo".to_string()), } ); } diff --git a/src/query/mod.rs b/src/query/mod.rs index 993a2cb..9ae0c7c 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -22,7 +22,7 @@ //! use rust_memex::query::{detect_intent, QueryIntent, QueryRouter}; //! //! // Quick intent detection -//! let intent = detect_intent("when did we buy dragon"); +//! let intent = detect_intent("when did we buy laptop"); //! assert!(matches!(intent, QueryIntent::Temporal)); //! //! // Full routing with recommendations diff --git a/src/query/router.rs b/src/query/router.rs index af9ce08..69d211b 100644 --- a/src/query/router.rs +++ b/src/query/router.rs @@ -15,7 +15,7 @@ //! ```rust //! use rust_memex::query::{QueryIntent, detect_intent, QueryRouter, RoutingDecision}; //! -//! let intent = detect_intent("when did we buy dragon"); +//! let intent = detect_intent("when did we buy laptop"); //! assert!(matches!(intent, QueryIntent::Temporal)); //! //! let router = QueryRouter::new(); @@ -237,7 +237,7 @@ fn semantic_keywords() -> &'static HashSet<&'static str> { /// ```rust /// use rust_memex::query::detect_intent; /// -/// let intent = detect_intent("when did we buy dragon"); +/// let intent = detect_intent("when did we buy laptop"); /// // Returns QueryIntent::Temporal /// ``` pub fn detect_intent(query: &str) -> QueryIntent { @@ -519,7 +519,7 @@ mod tests { #[test] fn test_temporal_intent_detection() { assert_eq!( - detect_intent("when did we buy dragon"), + detect_intent("when did we buy laptop"), QueryIntent::Temporal ); assert_eq!( @@ -568,12 +568,12 @@ mod tests { #[test] fn test_exact_intent_detection() { assert_eq!(detect_intent(r#"find "exact phrase""#), QueryIntent::Exact); - assert_eq!(detect_intent("search 'dragon'"), QueryIntent::Exact); + assert_eq!(detect_intent("search 'laptop'"), QueryIntent::Exact); } #[test] fn test_hybrid_default() { - assert_eq!(detect_intent("dragon mac studio"), QueryIntent::Hybrid); + assert_eq!(detect_intent("fast mac studio"), QueryIntent::Hybrid); assert_eq!(detect_intent("how to configure memex"), QueryIntent::Hybrid); } @@ -581,7 +581,7 @@ mod tests { fn test_router_confidence() { let router = QueryRouter::new(); - let decision = router.route("when exactly did we buy dragon in 2024"); + let decision = router.route("when exactly did we buy laptop in 2024"); assert_eq!(decision.intent, QueryIntent::Temporal); assert!(decision.confidence > 0.5); diff --git a/src/rag/mod.rs b/src/rag/mod.rs index a9fd1e0..d0dbe2e 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -644,7 +644,7 @@ fn extract_keywords(text: &str, max_keywords: usize) -> Vec { // Sort by frequency and take top N let mut words: Vec<_> = word_counts.into_iter().collect(); - words.sort_by(|a, b| b.1.cmp(&a.1)); + words.sort_by_key(|b| std::cmp::Reverse(b.1)); words .into_iter() @@ -3791,7 +3791,7 @@ mod tests { #[test] fn keyword_extraction_splits_paths_and_filters_session_tokens() { let keywords = extract_keywords( - "/Users/silver/Git/tools/TwinSweep session 2ff4de8b9a4e1234567890abcdef notes", + "/Users/tester/projects/TwinSweep session 2ff4de8b9a4e1234567890abcdef notes", 10, ); @@ -3802,24 +3802,24 @@ mod tests { #[test] fn search_options_can_carry_project_filter() { - let options = SearchOptions::deep().with_project(Some("Vista".to_string())); + let options = SearchOptions::deep().with_project(Some("Demo".to_string())); assert_eq!(options.layer_filter, None); - assert_eq!(options.project_filter.as_deref(), Some("Vista")); + assert_eq!(options.project_filter.as_deref(), Some("Demo")); } #[test] fn project_match_uses_metadata_fields() { assert!(metadata_matches_project( - &json!({"project": "Vista"}), - "vista" + &json!({"project": "Demo"}), + "demo" )); assert!(metadata_matches_project( &json!({"project_id": "Loctree"}), - "vetcoders" + "loctree" )); assert!(!metadata_matches_project( &json!({"project": "rust-memex"}), - "vista" + "demo" )); assert_eq!( SearchOptions::default().layer_filter, @@ -3833,14 +3833,14 @@ mod tests { [signals] Results: -- AICX lookup działa +- Config lookup succeeded [/signals] -[09:14:00] assistant: Tak, i to właśnie jest sedno: `aicx-dragon` to żywy endpoint MCP. -[09:15:33] user: ziom ale ty sobie sam skonfigurowałeś ~/.codex/config.toml -[09:15:47] assistant: Sprawdzam teraz lokalny kontrakt konfiguracji MCP dla Codexa. +[09:14:00] assistant: Right, that is the crux: the MCP endpoint is live and reachable. +[09:15:33] user: but you configured ~/.codex/config.toml yourself +[09:15:47] assistant: Checking the local MCP configuration contract for Codex now. [09:15:55] reasoning: **Checking config contract** -[09:16:06] assistant: Składnia configu wygląda już poprawnie według samego Codexa. +[09:16:06] assistant: The config syntax already looks correct per Codex itself. "#; let docs = extract_markdown_transcript_documents(raw, Path::new("sample.md")) diff --git a/src/search/hybrid.rs b/src/search/hybrid.rs index d5f63f6..83f89cb 100644 --- a/src/search/hybrid.rs +++ b/src/search/hybrid.rs @@ -906,17 +906,14 @@ mod tests { #[test] fn project_filter_matches_project_and_project_id() { - assert!(matches_project_filter( - &json!({"project": "Vista"}), - "vista" - )); + assert!(matches_project_filter(&json!({"project": "Demo"}), "demo")); assert!(matches_project_filter( &json!({"project_id": "Loctree"}), - "vetcoders" + "loctree" )); assert!(!matches_project_filter( &json!({"project": "rust-memex"}), - "vista" + "demo" )); } diff --git a/src/tests/memory.rs b/src/tests/memory.rs index e15e1ac..148c2d7 100644 --- a/src/tests/memory.rs +++ b/src/tests/memory.rs @@ -495,7 +495,7 @@ async fn memory_upsert_replaces_onion_family_by_original_id() -> Result<()> { let namespace = "memory-upsert-onion"; let initial_text = "This is a long document about indexing and embeddings. It exists to force onion slicing during the first upsert. \ The content is intentionally verbose and repetitive so the outer and core layers both exist and can later be replaced cleanly. \ -Project Vista keeps appearing here to provide a stable metadata anchor for search filters."; +Project Demo keeps appearing here to provide a stable metadata anchor for search filters."; let updated_text = "This is an updated long document about project search filters, namespace cache invalidation, and release hardening. \ It should fully replace the previous onion family instead of appending stale slices. \ The text stays comfortably above the slicing threshold so onion-fast still writes both outer and core documents."; @@ -504,7 +504,7 @@ The text stays comfortably above the slicing threshold so onion-fast still write namespace, "doc-1".to_string(), initial_text.to_string(), - json!({"slice_mode": "onion-fast", "project": "Vista"}), + json!({"slice_mode": "onion-fast", "project": "Demo"}), ) .await?; @@ -534,7 +534,7 @@ The text stays comfortably above the slicing threshold so onion-fast still write namespace, "doc-1".to_string(), updated_text.to_string(), - json!({"slice_mode": "onion-fast", "project": "Vista"}), + json!({"slice_mode": "onion-fast", "project": "Demo"}), ) .await?; diff --git a/src/tui/app.rs b/src/tui/app.rs index 048eb76..3a432a6 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -436,7 +436,7 @@ fn get_hostname() -> String { pub enum DbPathMode { /// Single shared path (e.g., ~/.ai-memories/lancedb) Shared, - /// Per-host path with hostname suffix (e.g., ~/.ai-memories/lancedb.dragon) + /// Per-host path with hostname suffix (e.g., ~/.ai-memories/lancedb.host-a) PerHost, } @@ -1136,12 +1136,11 @@ impl App { KeyCode::Up | KeyCode::Char('k') => self.handle_up(), KeyCode::Down | KeyCode::Char('j') => self.handle_down(), KeyCode::Char(' ') => self.handle_space(), - KeyCode::Char('r') => { + KeyCode::Char('r') // Retry health check - if self.step == WizardStep::HealthCheck && !self.health_running { + if self.step == WizardStep::HealthCheck && !self.health_running => { self.trigger_health_check(); } - } _ => {} } } @@ -1247,15 +1246,11 @@ impl App { self.editing_field = Some(self.focus); self.input_buffer = self.get_field_value(self.focus); } - WizardStep::HostSelection => { - if self.focus < self.hosts.len() { - self.toggle_host(self.focus); - } + WizardStep::HostSelection if self.focus < self.hosts.len() => { + self.toggle_host(self.focus); } - WizardStep::HealthCheck => { - if !self.health_running { - self.trigger_health_check(); - } + WizardStep::HealthCheck if !self.health_running => { + self.trigger_health_check(); } WizardStep::DataSetup => { self.handle_data_setup_enter(); diff --git a/src/tui/detection.rs b/src/tui/detection.rs index 8371073..e01d108 100644 --- a/src/tui/detection.rs +++ b/src/tui/detection.rs @@ -158,11 +158,6 @@ pub async fn detect_providers() -> Vec { providers.push(provider); } - // Check dragon:12345 (common remote MLX server) - if let Some(provider) = detect_mlx(&client, "http://dragon", 12345).await { - providers.push(provider); - } - providers } diff --git a/src/tui/monitor.rs b/src/tui/monitor.rs index 0d1d897..7ed1f3c 100644 --- a/src/tui/monitor.rs +++ b/src/tui/monitor.rs @@ -147,6 +147,10 @@ struct GpuMetrics { memory_total: Option, } +// On non-macOS targets the cfg'd early return makes the macOS ioreg body below +// unreachable by design; allow it so `-D warnings` stays green cross-platform +// without dead-coding the macOS-only helpers. +#[cfg_attr(not(target_os = "macos"), allow(unreachable_code))] fn probe_gpu() -> Result { #[cfg(not(target_os = "macos"))] return Err(GpuStatus::Unavailable {