From ed56eb6c6eb7d99a7d933e65fdf5e15608cc00d0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Jul 2026 09:22:25 +0200 Subject: [PATCH 1/3] fix: harden configuration and runtime quality --- .env.example | 12 +- .github/workflows/verify.yml | 10 + .gitignore | 5 + README.md | 22 +- config/settings.yaml | 9 +- docker-compose.yml | 8 +- docs/INSTALLATION.md | 4 +- install.sh | 11 +- openbrain.egg-info/PKG-INFO | 470 ---------------------- openbrain.egg-info/SOURCES.txt | 51 --- openbrain.egg-info/dependency_links.txt | 1 - openbrain.egg-info/entry_points.txt | 2 - openbrain.egg-info/requires.txt | 21 - openbrain.egg-info/top_level.txt | 4 - pyproject.toml | 5 +- scripts/check_db.py | 2 +- scripts/import_sample.py | 24 +- src/__init__.py | 8 +- src/analytics/trends.py | 11 +- src/analytics/weekly_report.py | 19 +- src/api/compaction.py | 2 +- src/api/consolidation.py | 14 +- src/api/context.py | 8 +- src/api/continuity.py | 20 +- src/api/imports.py | 14 +- src/api/lifecycle.py | 17 +- src/api/main.py | 34 +- src/api/maintenance.py | 5 +- src/api/operations.py | 2 +- src/api/pruning.py | 17 +- src/api/scopes.py | 4 +- src/cli/__init__.py | 30 +- src/cli/import_data.py | 2 +- src/cli/maintenance.py | 5 +- src/cli/serve.py | 4 +- src/cli/store.py | 3 +- src/connectors/claude_code.py | 4 +- src/connectors/file_watcher.py | 2 - src/connectors/gmail.py | 15 +- src/connectors/telegram.py | 4 +- src/connectors/whatsapp.py | 17 +- src/context/builder.py | 14 +- src/context/cache.py | 8 +- src/db/compaction_queries.py | 23 +- src/db/connection.py | 7 +- src/db/consolidation_queries.py | 15 +- src/db/context_queries.py | 5 +- src/db/continuity_queries.py | 2 +- src/db/import_queries.py | 4 +- src/db/lifecycle_queries.py | 15 +- src/db/migrate.py | 33 ++ src/db/pruning_queries.py | 15 +- src/db/queries.py | 2 - src/db/scope_queries.py | 4 +- src/embedder/__init__.py | 6 +- src/extractors/entities.py | 2 +- src/extractors/tagger.py | 2 +- src/importers/hermes_context.py | 4 +- src/importers/hermes_markdown.py | 2 +- src/importers/hermes_operations.py | 2 +- src/importers/hermes_sessions.py | 2 +- src/importers/providers.py | 2 +- src/importers/runner.py | 6 +- src/importers/staging.py | 2 +- src/ingestion/importer.py | 4 +- src/main.py | 18 +- src/maintenance/orchestrator.py | 17 +- src/notifications/email_notifier.py | 2 +- src/notifications/telegram_bot.py | 6 +- src/openbrain_hermes_plugin/README.md | 3 +- src/openbrain_hermes_plugin/__init__.py | 58 ++- src/openbrain_hermes_plugin/plugin.yaml | 2 +- src/providers/host_config.py | 5 +- src/runtime_config.py | 181 +++++++++ src/sandbox/__init__.py | 4 +- src/sandbox/cli.py | 4 +- src/sandbox/docker_sandbox.py | 109 +++-- src/setup/__init__.py | 160 ++++++-- src/version.py | 22 + tests/test_agent_attribution.py | 27 +- tests/test_code_quality_regressions.py | 245 +++++++++++ tests/test_continuity_models.py | 2 +- tests/test_core.py | 43 +- tests/test_extended_hermes_importers.py | 6 +- tests/test_hermes_markdown_importer.py | 4 +- tests/test_hermes_plugin_install.py | 1 - tests/test_hermes_provider_contract.py | 39 ++ tests/test_host_adapters.py | 28 +- tests/test_import_rollback.py | 7 +- tests/test_import_runner.py | 34 +- tests/test_installation_productization.py | 7 +- tests/test_proposal_api_contracts.py | 2 +- tests/test_release_readiness.py | 2 +- 93 files changed, 1121 insertions(+), 1020 deletions(-) delete mode 100644 openbrain.egg-info/PKG-INFO delete mode 100644 openbrain.egg-info/SOURCES.txt delete mode 100644 openbrain.egg-info/dependency_links.txt delete mode 100644 openbrain.egg-info/entry_points.txt delete mode 100644 openbrain.egg-info/requires.txt delete mode 100644 openbrain.egg-info/top_level.txt create mode 100644 src/runtime_config.py create mode 100644 src/version.py create mode 100644 tests/test_code_quality_regressions.py diff --git a/.env.example b/.env.example index e13f384..9af76c4 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,8 @@ # DATABASE (Required) DB_PASSWORD=your_secure_database_password DB_HOST=localhost -DB_PORT=5432 +DB_PORT=5433 +DB_HOST_PORT=5433 DB_NAME=openbrain DB_USER=postgres DB_TIMEZONE=auto @@ -14,7 +15,7 @@ DB_TIMEZONE=auto # Choose: openrouter, openai, ollama, custom EMBEDDER_PROVIDER=openrouter -# OpenRouter (FREE - recommended) +# OpenRouter (hosted; API key required) OPENROUTER_API_KEY=your_openrouter_api_key # OR OpenAI @@ -27,15 +28,16 @@ OPENROUTER_API_KEY=your_openrouter_api_key # CUSTOM_API_URL=https://api.example.com/v1 # CUSTOM_API_KEY=your_custom_api_key -# PORTS -MCP_PORT=8080 +# SERVICE PORTS +# MCP uses stdio and does not listen on a TCP port. API_PORT=8000 DASHBOARD_PORT=8501 # API OPERATIONS # production enables authentication by default and refuses to start without a key. OPENBRAIN_ENV=development -# OPENBRAIN_API_KEY=replace-with-a-long-random-secret +# Generated automatically by: openbrain configure --project-root . +# OPENBRAIN_API_KEY=managed-automatically # OPENBRAIN_AUTH_REQUIRED=true OPENBRAIN_MAX_REQUEST_BYTES=1048576 OPENBRAIN_RATE_LIMIT_REQUESTS=120 diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 639e1c6..0dc0087 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -36,6 +36,16 @@ jobs: cache: pip - name: Install run: pip install -e '.[dev]' + - name: Validate generated environment and container configuration + env: + OPENBRAIN_CONFIG_DIR: ${{ runner.temp }}/openbrain-config + run: | + python -m src.cli configure --project-root . + docker compose config --quiet + - name: Compile shipped modules + run: python -m compileall -q src scripts tests + - name: Lint correctness rules + run: ruff check src scripts tests --select E9,F401,F541,F63,F7,F82,E722,F841,S110,ASYNC221,PLW1508,PLW1510,RUF012,B017,DTZ001,DTZ005,DTZ007 - name: Apply migrations run: python scripts/migrate.py - name: Test diff --git a/.gitignore b/.gitignore index 00f2d38..c5c5eb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ __pycache__/ *.pyc .venv/ +*.egg-info/ +build/ +dist/ +.pytest-*/ +.verification/ diff --git a/README.md b/README.md index 81102b1..3101c47 100644 --- a/README.md +++ b/README.md @@ -95,10 +95,10 @@ Integration guides: Open Brain requires Python 3.11+ and PostgreSQL with pgvector. -For a reproducible v1.0.0 installation, review and run the release-pinned installer: +Review and run the current installer: ```bash -curl -fsSL https://raw.githubusercontent.com/benclawbot/open-brain/v1.0.0/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/benclawbot/open-brain/master/install.sh | sh ``` Verify: @@ -108,13 +108,12 @@ openbrain --version openbrain --help ``` -The installer uses `pipx`, keeping Open Brain isolated from system Python packages. +The installer uses `pipx`, keeping Open Brain isolated from system Python packages. It also generates a private OpenBrain API key once in `~/.config/openbrain/.env`; upgrades reuse that key. ### Hermes ```bash openbrain install-hermes -export OPENBRAIN_URL=http://127.0.0.1:8000 hermes memory setup ``` @@ -144,15 +143,16 @@ pip install -e '.[dev]' ```env DB_HOST=localhost -DB_PORT=5432 +DB_PORT=5433 +DB_HOST_PORT=5433 DB_NAME=openbrain DB_USER=postgres -DB_PASSWORD=change-me +DB_PASSWORD=replace-with-a-random-password DB_TIMEZONE=auto ``` ```bash -cp .env.example .env +openbrain configure --project-root . docker compose up -d --build docker compose ps ``` @@ -168,6 +168,14 @@ The API container applies pending migrations during startup. Containers reach Po Already-applied migrations must never be edited. Add a new migration instead. +## API keys + +`OPENBRAIN_API_KEY` protects the OpenBrain HTTP API and the memories stored behind it. It is OpenBrain's own shared secretβ€”not a key issued by OpenAI, OpenRouter, or another model provider. The installer generates it automatically, stores it in the private per-user environment file, and the API, provider SDK, host adapters, and Hermes plugin load it transparently. `openbrain configure --project-root .` copies the same key into the project's private `.env` and generates its database password. Existing credentials are preserved on upgrades and repeated configuration runs. + +Production mode (`OPENBRAIN_ENV=production`) requires authentication and refuses to start without this key. You normally do not need to view or copy it. Explicit environment variables still take precedence when connecting to a remote OpenBrain deployment. + +Embedding-provider keys are separate. Set `OPENROUTER_API_KEY` or `OPENAI_API_KEY` only when using that hosted provider for semantic embeddings. Create those keys in the provider's account dashboard. A local Ollama server does not require an API key. + ## CLI ```bash diff --git a/config/settings.yaml b/config/settings.yaml index 392df66..3049162 100644 --- a/config/settings.yaml +++ b/config/settings.yaml @@ -3,7 +3,6 @@ database: port: 5433 name: openbrain user: postgres - password: openbrain timezone: auto embedder: @@ -13,14 +12,14 @@ embedder: ollama_base_url: http://localhost:11434 mcp: - host: 0.0.0.0 - port: 8080 + transport: stdio api: - host: 0.0.0.0 + host: 127.0.0.1 port: 8000 cors_origins: - - "*" + - http://localhost:8501 + - http://localhost:8000 dashboard: port: 8501 diff --git a/docker-compose.yml b/docker-compose.yml index 2bc3d46..68a3cf7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: environment: DB_HOST: postgres DB_NAME: openbrain - DB_PASSWORD: openbrain + DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD must be set in .env} DB_PORT: 5432 DB_USER: postgres OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-http://host.docker.internal:11434} @@ -27,7 +27,7 @@ services: environment: DB_HOST: postgres DB_NAME: openbrain - DB_PASSWORD: openbrain + DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD must be set in .env} DB_PORT: 5432 DB_USER: postgres STREAMLIT_PORT: 8501 @@ -39,7 +39,7 @@ services: container_name: openbrain-postgres environment: POSTGRES_DB: openbrain - POSTGRES_PASSWORD: openbrain + POSTGRES_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD must be set in .env} POSTGRES_USER: postgres healthcheck: interval: 5s @@ -50,7 +50,7 @@ services: timeout: 5s image: pgvector/pgvector:pg16 ports: - - 5433:5432 + - "127.0.0.1:${DB_HOST_PORT:-5433}:5432" volumes: - postgres_data:/var/lib/postgresql/data volumes: diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 55a71e5..403dc29 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -6,7 +6,7 @@ curl -fsSL https://raw.githubusercontent.com/benclawbot/open-brain/master/install.sh | sh ``` -The installer verifies Python 3.11+, installs or upgrades Open Brain through `pipx`, detects Hermes, installs the bundled Hermes provider when Hermes is present, and runs `openbrain doctor`. +The installer verifies Python 3.11+, installs or upgrades Open Brain through `pipx`, generates a private API key in `~/.config/openbrain/.env`, detects Hermes, installs the bundled Hermes provider when Hermes is present, and runs `openbrain doctor`. Repeated installs preserve the existing key. Set `OPENBRAIN_INSTALL_HERMES=0` to skip automatic Hermes wiring. Set `OPENBRAIN_REPO_URL` to install from a fork. @@ -18,7 +18,7 @@ Automatic installation copies the packaged provider into `${HERMES_HOME:-~/.herm openbrain install-hermes --force ``` -Then set `OPENBRAIN_URL` and select `openbrain` from `hermes memory setup`. +Then select `openbrain` from `hermes memory setup`. The local URL and generated API key load automatically. ## Other coding agents diff --git a/install.sh b/install.sh index 390dbbc..dfbaf49 100644 --- a/install.sh +++ b/install.sh @@ -30,11 +30,18 @@ else "$PIPX_BIN" install "git+$REPO_URL" fi +OPENBRAIN_BIN="$(command -v openbrain || true)" +if [ -z "$OPENBRAIN_BIN" ]; then + OPENBRAIN_BIN="${PIPX_BIN_DIR:-$HOME/.local/bin}/openbrain" +fi + +"$OPENBRAIN_BIN" configure + if [ "$HERMES_MODE" = "1" ] || [ "$HERMES_MODE" = "true" ] || { [ "$HERMES_MODE" = "auto" ] && command -v hermes >/dev/null 2>&1; }; then - openbrain install-hermes --force + "$OPENBRAIN_BIN" install-hermes --force fi -openbrain doctor +"$OPENBRAIN_BIN" doctor cat <<'EOF' Open Brain installed. diff --git a/openbrain.egg-info/PKG-INFO b/openbrain.egg-info/PKG-INFO deleted file mode 100644 index 2843faa..0000000 --- a/openbrain.egg-info/PKG-INFO +++ /dev/null @@ -1,470 +0,0 @@ -Metadata-Version: 2.4 -Name: openbrain -Version: 0.1.0 -Summary: Standalone personal memory management system with semantic search -Author-email: Tom -License: MIT -Keywords: memory,embeddings,semantic-search,ai,postgres,pgvector -Classifier: Development Status :: 3 - Alpha -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Requires-Python: >=3.11 -Description-Content-Type: text/markdown -Requires-Dist: psycopg2-binary>=2.9.9 -Requires-Dist: sqlalchemy>=2.0.0 -Requires-Dist: asyncpg>=0.29.0 -Requires-Dist: pgvector>=0.2.0 -Requires-Dist: pyyaml>=6.0.1 -Requires-Dist: python-dotenv>=1.0.0 -Requires-Dist: fastapi>=0.109.0 -Requires-Dist: uvicorn>=0.27.0 -Requires-Dist: streamlit>=1.31.0 -Requires-Dist: pandas>=2.2.0 -Requires-Dist: matplotlib>=3.8.0 -Requires-Dist: requests>=2.31.0 -Requires-Dist: httpx>=0.26.0 -Requires-Dist: pydantic>=2.5.0 -Requires-Dist: pydantic-settings>=2.1.0 -Provides-Extra: dev -Requires-Dist: pytest>=7.4.0; extra == "dev" -Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev" -Requires-Dist: black>=24.1.0; extra == "dev" -Requires-Dist: ruff>=0.2.0; extra == "dev" - -# 🧠 Open Brain - -> Personal semantic memory system with MCP interface. Store, search, and analyze everything that matters to you. - -[![GitHub Stars](https://img.shields.io/github/stars/benclawbot/open-brain)](https://github.com/benclawbot/open-brain/stargazers) -[![Docker](https://img.shields.io/docker/pulls/benclawbot/open-brain)](https://hub.docker.com/r/benclawbot/open-brain) -[![License](https://img.shields.io/github/license/benclawbot/open-brain)](https://github.com/benclawbot/open-brain/blob/master/LICENSE) - -## What is Open Brain? - -Open Brain is a **personal knowledge management system** that acts as your second brain. It: - -- πŸ“₯ **Ingests** data from anywhere (Telegram, WhatsApp, Claude Code, Gmail, files) -- 🧠 **Embeds** everything semantically (OpenRouter, OpenAI, Ollama, or any custom API) -- πŸ” **Searches** instantly using vector similarity -- πŸ“Š **Analyzes** trends, clusters, and connections -- πŸ”” **Notifies** you of important changes -- 🌐 **Serves** via MCP, REST API, CLI, or Dashboard - -Think of it as **Obsidian meets ChatGPT memory** β€” but accessible from any tool. - ---- - -## ✨ Features - -### Core -- **Semantic Search** β€” Find memories by meaning, not just keywords -- **Auto-Tagging** β€” Automatic topic and entity extraction -- **Entity Recognition** β€” Extracts people, places, organizations, dates -- **Trend Analysis** β€” See what topics are emerging or declining - -### Integrations -- **MCP Server** β€” Use from Claude, Codex, or any MCP client -- **REST API** β€” HTTP access for any application -- **CLI** β€” Command-line interface for quick operations -- **Source Connectors** β€” Import from Telegram, WhatsApp, Gmail, Claude Code - -### UI -- **Streamlit Dashboard** β€” Visualize memories, stats, and trends -- **Weekly Reports** β€” Automated markdown reports - ---- - -## πŸš€ Quick Start - -### Prerequisites - -- [Docker](https://docker.com) + Docker Compose -- At least 2GB RAM -- (Optional) OpenRouter API key for embeddings - -### 1. Clone & Configure - -```bash -git clone https://github.com/benclawbot/open-brain.git -cd open-brain - -# Copy environment file -cp .env.example .env -``` - -### 2. Set Environment Variables - -Edit `.env`: - -```env -# Database -DB_PASSWORD=your_secure_password - -# Embeddings (OpenRouter = FREE) -OPENROUTER_API_KEY=your_openrouter_key - -# Optional: Telegram notifications -TELEGRAM_BOT_TOKEN= -TELEGRAM_CHAT_ID= -``` - -> **No API key?** OpenRouter has a free tier. Just sign up at [openrouter.ai](https://openrouter.ai). - -### 3. Start Everything - -```bash -docker compose up -d -``` - -### 4. Access Services - -| Service | URL | Description | -|---------|-----|-------------| -| **Dashboard** | http://localhost:8501 | Streamlit UI | -| **MCP Server** | http://localhost:8080 | MCP protocol | -| **REST API** | http://localhost:8000 | HTTP API | -| **API Docs** | http://localhost:8000/docs | Swagger docs | - ---- - -## πŸ”§ Configuration - -All settings in `config/settings.yaml`: - -```yaml -database: - host: postgres - port: 5432 - name: openbrain - user: postgres - password: ${DB_PASSWORD} - -embedder: - # Providers: openrouter, openai, ollama, custom - provider: openrouter - model: text-embedding-3-small - dimensions: 768 - -mcp: - host: 0.0.0.0 - port: 8080 - -api: - host: 0.0.0.0 - port: 8000 - -dashboard: - port: 8501 -``` - -### Embedder Providers - -| Provider | Env Variable | Notes | -|----------|-------------|-------| -| **OpenRouter** (default) | `OPENROUTER_API_KEY` | Free tier available | -| OpenAI | `OPENAI_API_KEY` | Paid | -| Ollama | `OLLAMA_BASE_URL` | Local, free | -| Custom | `CUSTOM_API_URL` + `CUSTOM_API_KEY` | Any OpenAI-compatible | - ---- - -## πŸ“‘ Usage - -### CLI - -```bash -# Install -pip install -e . - -# Search memories -openbrain search "what did I learn about AI" - -# Store a memory -openbrain store "Meeting with Oliver about trading bot" --source telegram --tags ai,trading - -# Show stats -openbrain stats - -# Generate weekly report -openbrain report - -# Start API server -openbrain serve -``` - -### MCP Tools - -Connect any MCP client to `http://localhost:8080`: - -```python -# Example: Using memory_search -{ - "name": "memory_search", - "arguments": { - "query": "trading strategies", - "limit": 5, - "sources": ["telegram", "claude"] - } -} -``` - -### REST API - -```bash -# Search -curl -X POST http://localhost:8000/memories/search \ - -H "Content-Type: application/json" \ - -d '{"query": "AI agents", "limit": 5}' - -# Store -curl -X POST http://localhost:8000/memories \ - -H "Content-Type: application/json" \ - -d '{"content": "New idea", "source": "manual"}' - -# Stats -curl http://localhost:8000/stats -``` - ---- - -## πŸ— Architecture - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Clients β”‚ -β”‚ Claude Code | Codex | OpenClaw | Custom Apps | CLI β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ MCP / HTTP / CLI - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Open Brain β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ MCP API β”‚ β”‚ REST API β”‚ β”‚ CLI Tools β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Application Layer β”‚ β”‚ -β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ -β”‚ β”‚ β”‚Extractorsβ”‚ β”‚ Tagger β”‚ β”‚Analytics β”‚ β”‚ Notifier β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ (Entitiesβ”‚ β”‚ (Auto-tagβ”‚ β”‚ (Trends) β”‚ β”‚(Telegram)β”‚ β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Embedder (Multi-Provider) β”‚ β”‚ -β”‚ β”‚ OpenRouter | OpenAI | Ollama | Custom β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ PostgreSQL + pgvector β”‚ -β”‚ memory table with vector embeddings, GIN indexes β”‚ -β”‚ for tags/entities, IVFFlat for similarity search β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Components - -| Component | Technology | Purpose | -|-----------|------------|---------| -| Database | PostgreSQL + pgvector | Storage + vector search | -| MCP Server | FastMCP | Tool interface for AI agents | -| REST API | FastAPI | HTTP access | -| CLI | Click | Terminal commands | -| Dashboard | Streamlit | Visual UI | -| Embedder | requests | Multi-provider embeddings | -| Extractors | NLTK/spaCy | Entity extraction | - ---- - -## πŸ“ Project Structure - -``` -open-brain/ -β”œβ”€β”€ config/ -β”‚ └── settings.yaml # Configuration -β”œβ”€β”€ src/ -β”‚ β”œβ”€β”€ main.py # MCP server entry -β”‚ β”œβ”€β”€ db/ # Database layer -β”‚ β”‚ β”œβ”€β”€ schema.sql -β”‚ β”‚ β”œβ”€β”€ connection.py -β”‚ β”‚ └── queries.py -β”‚ β”œβ”€β”€ embedder/ # Multi-provider embeddings -β”‚ β”œβ”€β”€ extractors/ # NER + tagging -β”‚ β”œβ”€β”€ analytics/ # Trends + reports -β”‚ β”œβ”€β”€ connectors/ # Source importers -β”‚ β”œβ”€β”€ cli/ # CLI commands -β”‚ β”œβ”€β”€ api/ # REST API -β”‚ β”œβ”€β”€ notifications/ # Telegram + email -β”‚ └── ingestion/ # Bulk import -β”œβ”€β”€ ui/ -β”‚ └── dashboard.py # Streamlit dashboard -β”œβ”€β”€ scripts/ -β”‚ β”œβ”€β”€ setup_db.py # Database setup -β”‚ β”œβ”€β”€ backup.sh # Automated backups -β”‚ └── healthcheck.sh # Health monitoring -β”œβ”€β”€ tests/ -β”‚ └── test_core.py -β”œβ”€β”€ docker-compose.yml # Full stack -β”œβ”€β”€ Dockerfile # App container -β”œβ”€β”€ pyproject.toml # CLI package -β”œβ”€β”€ requirements.txt # Python deps -└── README.md -``` - ---- - -## πŸ”Œ Source Connectors - -### Telegram - -```python -from src.connectors.telegram import TelegramImporter - -importer = TelegramImporter( - export_file="telegram_export.json" -) -importer.import_all(db_conn) -``` - -### WhatsApp - -```python -from src.connectors.whatsapp import WhatsAppImporter - -importer = WhatsAppImporter( - export_file="whatsapp_chat.txt" -) -importer.import_all(db_conn) -``` - -### Claude Code - -```python -from src.connectors.claude_code import ClaudeCodeImporter - -importer = ClaudeCodeImporter( - sessions_path="~/.claude/sessions" -) -importer.import_all(db_conn) -``` - -### Gmail - -```python -from src.connectors.gmail import GmailImporter - -importer = GmailImporter( - takeout_path="./mail" -) -importer.import_all(db_conn) -``` - ---- - -## πŸ“Š Analytics - -### Trend Detection - -Automatically detects: -- **Emerging topics** β€” Tags increasing >50% vs baseline -- **Declining topics** β€” Tags dropping >30% -- **New entities** β€” People/places appearing for first time -- **Co-occurrence** β€” Topics that appear together - -### Weekly Reports - -Generated every Sunday via cron: - -```markdown -# Weekly Memory Report - -### Activity -- New memories: 47 -- By source: telegram: 23, claude: 15, manual: 9 - -### What's Hot -- ai: +200% (15 mentions) -- trading: +50% (8 mentions) - -### Insights -- You're researching AI agents heavily this week -- Oliver appeared 5 times β€” significant collaboration -``` - ---- - -## πŸ”” Notifications - -### Telegram Alerts - -- New emerging trends -- Weekly reports -- Memory stats summaries - -### Email - -- Daily digests -- Weekly reports -- Anomaly alerts - ---- - -## 🐳 Docker Services - -| Service | Image | Ports | -|---------|-------|-------| -| postgres | pgvector/pgvector:0.5.1 | 5432 | -| api | benclawbot/open-brain | 8000 | -| dashboard | benclawbot/open-brain | 8501 | -| mcp | benclawbot/open-brain | 8080 | - ---- - -## πŸ€– Phone Installation (ARM) - -For running on an old Android phone converted to Linux: - -```bash -# Use ARM-compatible images -docker-compose -f docker-compose.arm.yml up -d - -# Or build locally on phone -docker build --platform=linux/arm64 . -``` - -Resources: ~500MB RAM, ~1GB storage - ---- - -## πŸ§ͺ Testing - -```bash -# Run tests -pytest tests/ - -# Test MCP connection -python -c "from src.main import mcp; print(mcp)" -``` - ---- - -## πŸ“ License - -MIT License β€” do whatever you want with it. - ---- - -## πŸ™ Acknowledgments - -- [pgvector](https://github.com/pgvector/pgvector) β€” Vector similarity for PostgreSQL -- [FastMCP](https://github.com/jlowin/fastmcp) β€” MCP framework -- [OpenRouter](https://openrouter.ai) β€” Free embedding API - ---- - -## πŸ”— Links - -- [GitHub](https://github.com/benclawbot/open-brain) -- [Report Issues](https://github.com/benclawbot/open-brain/issues) -- [Discussions](https://github.com/benclawbot/open-brain/discussions) diff --git a/openbrain.egg-info/SOURCES.txt b/openbrain.egg-info/SOURCES.txt deleted file mode 100644 index b89de6e..0000000 --- a/openbrain.egg-info/SOURCES.txt +++ /dev/null @@ -1,51 +0,0 @@ -README.md -pyproject.toml -config/settings.yaml -openbrain.egg-info/PKG-INFO -openbrain.egg-info/SOURCES.txt -openbrain.egg-info/dependency_links.txt -openbrain.egg-info/entry_points.txt -openbrain.egg-info/requires.txt -openbrain.egg-info/top_level.txt -scripts/check_db.py -scripts/import_sample.py -scripts/setup_db.py -src/__init__.py -src/main.py -src/analytics/__init__.py -src/analytics/trends.py -src/analytics/weekly_report.py -src/api/__init__.py -src/api/main.py -src/cli/__init__.py -src/cli/import_data.py -src/cli/report.py -src/cli/search.py -src/cli/serve.py -src/cli/stats.py -src/cli/store.py -src/connectors/__init__.py -src/connectors/claude_code.py -src/connectors/file_watcher.py -src/connectors/gmail.py -src/connectors/telegram.py -src/connectors/whatsapp.py -src/db/__init__.py -src/db/connection.py -src/db/queries.py -src/embedder/__init__.py -src/extractors/__init__.py -src/extractors/entities.py -src/extractors/tagger.py -src/ingestion/__init__.py -src/ingestion/importer.py -src/notifications/__init__.py -src/notifications/email_notifier.py -src/notifications/telegram_bot.py -src/sandbox/__init__.py -src/sandbox/cli.py -src/sandbox/docker_sandbox.py -src/setup/__init__.py -tests/test_core.py -ui/dashboard.py -ui/pages/1_Settings.py \ No newline at end of file diff --git a/openbrain.egg-info/dependency_links.txt b/openbrain.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/openbrain.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/openbrain.egg-info/entry_points.txt b/openbrain.egg-info/entry_points.txt deleted file mode 100644 index 6f0e6d0..0000000 --- a/openbrain.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -openbrain = src.cli:main diff --git a/openbrain.egg-info/requires.txt b/openbrain.egg-info/requires.txt deleted file mode 100644 index 0a695ab..0000000 --- a/openbrain.egg-info/requires.txt +++ /dev/null @@ -1,21 +0,0 @@ -psycopg2-binary>=2.9.9 -sqlalchemy>=2.0.0 -asyncpg>=0.29.0 -pgvector>=0.2.0 -pyyaml>=6.0.1 -python-dotenv>=1.0.0 -fastapi>=0.109.0 -uvicorn>=0.27.0 -streamlit>=1.31.0 -pandas>=2.2.0 -matplotlib>=3.8.0 -requests>=2.31.0 -httpx>=0.26.0 -pydantic>=2.5.0 -pydantic-settings>=2.1.0 - -[dev] -pytest>=7.4.0 -pytest-asyncio>=0.23.0 -black>=24.1.0 -ruff>=0.2.0 diff --git a/openbrain.egg-info/top_level.txt b/openbrain.egg-info/top_level.txt deleted file mode 100644 index cb958df..0000000 --- a/openbrain.egg-info/top_level.txt +++ /dev/null @@ -1,4 +0,0 @@ -config -scripts -src -ui diff --git a/pyproject.toml b/pyproject.toml index 6f2e51a..96d3654 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "1.0.0" description = "Shared personal memory, agent continuity, and actionable context" readme = "README.md" requires-python = ">=3.11" -license = {text = "MIT"} +license = "MIT" authors = [ {name = "Ben Clawbot"} ] @@ -16,7 +16,6 @@ keywords = ["memory", "agents", "semantic-search", "mcp", "postgres", "pgvector" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] @@ -63,4 +62,4 @@ include = ["src*", "config*"] [tool.setuptools.package-data] config = ["*.yaml", "*.yml"] "src.db" = ["migrations/*.sql"] -"src.openbrain_hermes_plugin" = ["plugin.yaml", "README.md"] \ No newline at end of file +"src.openbrain_hermes_plugin" = ["plugin.yaml", "README.md"] diff --git a/scripts/check_db.py b/scripts/check_db.py index abc07ee..f147045 100755 --- a/scripts/check_db.py +++ b/scripts/check_db.py @@ -9,7 +9,7 @@ def check_db(): try: conn = psycopg2.connect( host=os.getenv('DB_HOST', 'postgres'), - port=int(os.getenv('DB_PORT', 5432)), + port=int(os.getenv('DB_PORT', '5432')), database=os.getenv('DB_NAME', 'openbrain'), user=os.getenv('DB_USER', 'postgres'), password=os.getenv('DB_PASSWORD', 'openbrain') diff --git a/scripts/import_sample.py b/scripts/import_sample.py index 9ed0ebc..0a0849d 100644 --- a/scripts/import_sample.py +++ b/scripts/import_sample.py @@ -2,17 +2,15 @@ """ Sample data import script for Open Brain. """ -import uuid -from datetime import datetime, timedelta +import logging +from datetime import datetime, timedelta, timezone -import sys -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) +from src.db import queries +from src.embedder import create_embedding +from src.extractors.entities import extract_entities +from src.extractors.tagger import auto_tag -from db import queries -from embedder import create_embedding -from extractors.entities import extract_entities -from extractors.tagger import auto_tag +logger = logging.getLogger(__name__) SAMPLE_MEMORIES = [ @@ -93,11 +91,11 @@ def import_samples(): embedding = None try: embedding = create_embedding(mem['content']) - except Exception: - pass + except Exception as exc: + logger.warning("Could not create embedding for sample memory: %s", exc) # Store memory - memory_id = queries.insert_memory( + queries.insert_memory( source=mem['source'], content=mem['content'], embedding=embedding, @@ -105,7 +103,7 @@ def import_samples(): tags=list(tags.keys()), tag_sources=tags, importance=mem['importance'], - original_date=datetime.now() - timedelta(days=i) + original_date=datetime.now(timezone.utc) - timedelta(days=i) ) print(f"βœ“ Imported: {mem['content'][:50]}...") diff --git a/src/__init__.py b/src/__init__.py index e1f5855..9f5016c 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1 +1,7 @@ -# Open Brain +"""Open Brain package.""" + +from src.version import get_version + +__version__ = get_version() + +__all__ = ["__version__"] diff --git a/src/analytics/trends.py b/src/analytics/trends.py index a79d01c..d972019 100644 --- a/src/analytics/trends.py +++ b/src/analytics/trends.py @@ -2,16 +2,9 @@ Trend detection for Open Brain. Analyzes memory patterns over time. """ -import sys -import os -from collections import Counter -from datetime import datetime, timedelta -from typing import Dict, List, Optional +from typing import Dict, List -# Add src to path for imports -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from db import queries +from ..db import queries class TrendAnalyzer: diff --git a/src/analytics/weekly_report.py b/src/analytics/weekly_report.py index 8f8721a..43b7371 100644 --- a/src/analytics/weekly_report.py +++ b/src/analytics/weekly_report.py @@ -2,16 +2,11 @@ Weekly report generation for Open Brain. Generates markdown reports of memory activity. """ -import sys -import os -from datetime import datetime, timedelta -from typing import Dict, List, Optional +from datetime import datetime, timezone +from typing import Dict, List -# Add src to path for imports -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from db import queries -from analytics.trends import TrendAnalyzer +from ..db import queries +from .trends import TrendAnalyzer class WeeklyReport: @@ -35,7 +30,7 @@ def generate(self) -> str: report_lines = [ "# πŸ“Š Open Brain Weekly Report", "", - f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + f"**Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}", f"**Period:** Last {self.days} days", "", "---", @@ -149,8 +144,8 @@ def generate_simple(self) -> str: memories = queries.get_memories_for_report(self.days) lines = [ - f"# Weekly Memory Report", - f"", + "# Weekly Memory Report", + "", f"Week: {stats['this_week']} | Month: {stats['this_month']} | Total: {stats['total']}", "", ] diff --git a/src/api/compaction.py b/src/api/compaction.py index 26889d3..0151e1d 100644 --- a/src/api/compaction.py +++ b/src/api/compaction.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, HTTPException from pydantic import BaseModel, Field -from db.compaction_queries import compact_events +from ..db.compaction_queries import compact_events router = APIRouter(prefix="/compaction", tags=["compaction"]) diff --git a/src/api/consolidation.py b/src/api/consolidation.py index 21fc6ed..cd3e776 100644 --- a/src/api/consolidation.py +++ b/src/api/consolidation.py @@ -6,18 +6,8 @@ from fastapi import APIRouter, HTTPException, Query, status from pydantic import BaseModel, Field -try: - from ..db.consolidation_queries import ( - apply_consolidation_proposal, - generate_consolidation_proposals, - list_consolidation_proposals, - resolve_consolidation_proposal, - reverse_consolidation_execution, - ) - from .proposals import ProposalActorRequest, ProposalReviewRequest -except ImportError: - from api.proposals import ProposalActorRequest, ProposalReviewRequest - from db.consolidation_queries import ( +from .proposals import ProposalActorRequest, ProposalReviewRequest +from ..db.consolidation_queries import ( apply_consolidation_proposal, generate_consolidation_proposals, list_consolidation_proposals, diff --git a/src/api/context.py b/src/api/context.py index 0c9c8e7..8af02a1 100644 --- a/src/api/context.py +++ b/src/api/context.py @@ -2,10 +2,10 @@ from fastapi import APIRouter, HTTPException, Query -from context.builder import build_context_packet -from context.cache import cleanup_context_cache, context_cache_stats -from context.models import ContextFeedback, ContextPacket, ContextRequest -from db.context_queries import save_context_feedback +from ..context.builder import build_context_packet +from ..context.cache import cleanup_context_cache, context_cache_stats +from ..context.models import ContextFeedback, ContextPacket, ContextRequest +from ..db.context_queries import save_context_feedback router = APIRouter(tags=["context"]) diff --git a/src/api/continuity.py b/src/api/continuity.py index ee7939b..f7752d8 100644 --- a/src/api/continuity.py +++ b/src/api/continuity.py @@ -2,16 +2,16 @@ from fastapi import APIRouter, HTTPException, status -from api.compaction import router as compaction_router -from api.consolidation import router as consolidation_router -from api.context import router as context_router -from api.imports import router as imports_router -from api.lifecycle import router as lifecycle_router -from api.maintenance import router as maintenance_router -from api.pruning import router as pruning_router -from api.scopes import router as scopes_router -from continuity.models import EventCreate, EventRecord -from db.continuity_queries import ingest_event +from .compaction import router as compaction_router +from .consolidation import router as consolidation_router +from .context import router as context_router +from .imports import router as imports_router +from .lifecycle import router as lifecycle_router +from .maintenance import router as maintenance_router +from .pruning import router as pruning_router +from .scopes import router as scopes_router +from ..continuity.models import EventCreate, EventRecord +from ..db.continuity_queries import ingest_event router = APIRouter(prefix="/v1", tags=["continuity"]) router.include_router(scopes_router) diff --git a/src/api/imports.py b/src/api/imports.py index 4e23be5..aaa640d 100644 --- a/src/api/imports.py +++ b/src/api/imports.py @@ -9,12 +9,12 @@ from fastapi import APIRouter, HTTPException from pydantic import BaseModel, ConfigDict, Field -from db.import_queries import rollback_staged_import -from importers.base import ImportSource -from importers.hermes_markdown import HermesMarkdownImporter -from importers.providers import provider_adapter, provider_descriptors -from importers.runner import ImportSummary, run_import -from importers.staging import seal_import_run +from ..db.import_queries import rollback_staged_import +from ..importers.base import ImportSource +from ..importers.hermes_markdown import HermesMarkdownImporter +from ..importers.providers import provider_adapter, provider_descriptors +from ..importers.runner import ImportSummary, run_import +from ..importers.staging import seal_import_run router = APIRouter(tags=["imports"]) @@ -165,4 +165,4 @@ async def rollback_import(run_id: UUID, request: ImportRollbackRequest) -> Impor except ValueError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc except Exception as exc: - raise HTTPException(status_code=500, detail=str(exc)) from exc \ No newline at end of file + raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/src/api/lifecycle.py b/src/api/lifecycle.py index 18a16fa..824d445 100644 --- a/src/api/lifecycle.py +++ b/src/api/lifecycle.py @@ -8,26 +8,15 @@ from fastapi import APIRouter, HTTPException, Query, status from pydantic import BaseModel, Field -try: - from ..db.lifecycle_queries import ( +from .proposals import ProposalActorRequest, ProposalReviewRequest +from ..db.lifecycle_queries import ( apply_lifecycle_proposal, generate_lifecycle_proposals, list_lifecycle_proposals, resolve_lifecycle_proposal, reverse_lifecycle_execution, ) - from ..lifecycle.execution import LifecycleExecutionError - from .proposals import ProposalActorRequest, ProposalReviewRequest -except ImportError: # Support legacy execution with src/ directly on sys.path. - from api.proposals import ProposalActorRequest, ProposalReviewRequest - from db.lifecycle_queries import ( - apply_lifecycle_proposal, - generate_lifecycle_proposals, - list_lifecycle_proposals, - resolve_lifecycle_proposal, - reverse_lifecycle_execution, - ) - from lifecycle.execution import LifecycleExecutionError +from ..lifecycle.execution import LifecycleExecutionError router = APIRouter(prefix="/lifecycle", tags=["lifecycle"]) diff --git a/src/api/main.py b/src/api/main.py index 68a4802..79dc760 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -10,30 +10,32 @@ from fastapi import FastAPI, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware + +from src.runtime_config import load_runtime_environment + +load_runtime_environment() from fastapi.responses import JSONResponse from pydantic import BaseModel, Field -# Add src to path -import sys -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from api.continuity import router as continuity_router -from api.operations import configure_operations, database_readiness -from db.connection import init_db -from db.attribution import ( +from .continuity import router as continuity_router +from .operations import configure_operations, database_readiness +from ..analytics.trends import TrendAnalyzer +from ..analytics.weekly_report import generate_weekly_report +from ..db.connection import init_db +from ..db.attribution import ( search_memories, get_memory_by_id, insert_memory, get_recent_memories, ) -from db.queries import get_memory_stats -from embedder import create_embedding -from extractors.entities import extract_entities -from extractors.tagger import get_tagger -from analytics.trends import TrendAnalyzer -from analytics.weekly_report import generate_weekly_report +from ..db.queries import get_memory_stats +from ..embedder import create_embedding +from ..extractors.entities import extract_entities +from ..extractors.tagger import get_tagger +from ..version import get_version logger = logging.getLogger("openbrain.api") +VERSION = get_version() @asynccontextmanager @@ -51,7 +53,7 @@ async def lifespan(app: FastAPI): app = FastAPI( title="Open Brain API", description="REST API for memory management and durable agent continuity", - version="0.2.0", + version=VERSION, lifespan=lifespan, ) configure_operations(app) @@ -108,7 +110,7 @@ async def root(): """Root endpoint.""" return { "name": "Open Brain API", - "version": "0.2.0", + "version": VERSION, "docs": "/docs", } diff --git a/src/api/maintenance.py b/src/api/maintenance.py index 70679f0..b2ae4c7 100644 --- a/src/api/maintenance.py +++ b/src/api/maintenance.py @@ -5,10 +5,7 @@ from fastapi import APIRouter from pydantic import BaseModel, Field -try: - from ..maintenance import MaintenanceOptions, run_maintenance -except ImportError: - from maintenance import MaintenanceOptions, run_maintenance +from ..maintenance import MaintenanceOptions, run_maintenance router = APIRouter(prefix="/maintenance", tags=["maintenance"]) diff --git a/src/api/operations.py b/src/api/operations.py index afe45dd..6c8c0e3 100644 --- a/src/api/operations.py +++ b/src/api/operations.py @@ -16,7 +16,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse -from db.connection import get_db_cursor +from ..db.connection import get_db_cursor logger = logging.getLogger("openbrain.api") diff --git a/src/api/pruning.py b/src/api/pruning.py index 23a85af..dcd5d18 100644 --- a/src/api/pruning.py +++ b/src/api/pruning.py @@ -6,26 +6,15 @@ from fastapi import APIRouter, HTTPException, Query, status from pydantic import BaseModel, Field -try: - from ..db.pruning_queries import ( +from .proposals import ProposalActorRequest, ProposalReviewRequest +from ..db.pruning_queries import ( apply_pruning_proposal, generate_pruning_proposals, list_pruning_proposals, restore_tombstone, review_pruning_proposal, ) - from ..pruning.execution import PruningConflict - from .proposals import ProposalActorRequest, ProposalReviewRequest -except ImportError: - from api.proposals import ProposalActorRequest, ProposalReviewRequest - from db.pruning_queries import ( - apply_pruning_proposal, - generate_pruning_proposals, - list_pruning_proposals, - restore_tombstone, - review_pruning_proposal, - ) - from pruning.execution import PruningConflict +from ..pruning.execution import PruningConflict router = APIRouter(prefix="/pruning", tags=["pruning"]) diff --git a/src/api/scopes.py b/src/api/scopes.py index 9a45958..581c854 100644 --- a/src/api/scopes.py +++ b/src/api/scopes.py @@ -5,8 +5,8 @@ from fastapi import APIRouter, HTTPException, status from pydantic import BaseModel, ConfigDict, Field -from continuity.scopes import IdentityRecord, IdentityRef, SessionOpen, SessionRecord -from db.scope_queries import close_session, open_session, resolve_identity +from ..continuity.scopes import IdentityRecord, IdentityRef, SessionOpen, SessionRecord +from ..db.scope_queries import close_session, open_session, resolve_identity router = APIRouter(tags=["scopes"]) diff --git a/src/cli/__init__.py b/src/cli/__init__.py index 39b82c9..2741d3f 100644 --- a/src/cli/__init__.py +++ b/src/cli/__init__.py @@ -7,21 +7,16 @@ import shutil import subprocess import sys -from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from dotenv import load_dotenv +from src.runtime_config import configure_runtime_environment, load_runtime_environment +from src.version import get_version -_dotenv_path = os.path.join(os.path.dirname(__file__), "..", "..", ".env") -if os.path.exists(_dotenv_path): - load_dotenv(_dotenv_path) +load_runtime_environment() def _version() -> str: - try: - return version("openbrain") - except PackageNotFoundError: - return "development" + return get_version() def _run(command: list[str]) -> None: @@ -65,7 +60,14 @@ def install_hermes_cmd(hermes_home: str | None = None, force: bool = False) -> i for name in required: shutil.copy2(source / name, destination / name) print(f"Installed the Open Brain Hermes provider at {destination}") - print("Set OPENBRAIN_URL, then run `hermes memory setup` and select `openbrain`.") + print("Run `hermes memory setup` and select `openbrain`; credentials load automatically.") + return 0 + + +def configure_cmd(project_root: str | None = None) -> int: + """Create or reuse private credentials without printing the secret.""" + path = configure_runtime_environment(Path(project_root) if project_root else None) + print(f"Open Brain credentials configured at {path}") return 0 @@ -107,6 +109,12 @@ def main() -> int: update_parser = subparsers.add_parser("update", help="Upgrade Open Brain and apply additive migrations") update_parser.add_argument("--skip-migrations", action="store_true") + configure_parser = subparsers.add_parser( + "configure", + help="Generate private local credentials", + ) + configure_parser.add_argument("--project-root") + hermes_parser = subparsers.add_parser("install-hermes", help="Install the native Hermes memory provider") hermes_parser.add_argument("--hermes-home") hermes_parser.add_argument("--force", action="store_true") @@ -151,6 +159,8 @@ def main() -> int: return serve_cmd(args) if args.command == "update": return update_cmd(skip_migrations=args.skip_migrations) + if args.command == "configure": + return configure_cmd(args.project_root) if args.command == "install-hermes": return install_hermes_cmd(args.hermes_home, args.force) if args.command == "maintenance": diff --git a/src/cli/import_data.py b/src/cli/import_data.py index 665fa91..348ee44 100644 --- a/src/cli/import_data.py +++ b/src/cli/import_data.py @@ -35,7 +35,7 @@ def import_cmd(args: Namespace) -> int: print(f"Unknown source: {source}") return 1 - print(f"\nImport complete!") + print("\nImport complete!") print(f"Imported: {result.get('imported', 0)}") print(f"Total processed: {result.get('total_processed', 0)}") diff --git a/src/cli/maintenance.py b/src/cli/maintenance.py index 27fd451..7b3b469 100644 --- a/src/cli/maintenance.py +++ b/src/cli/maintenance.py @@ -5,10 +5,7 @@ import json from uuid import UUID -try: - from ..maintenance import MaintenanceOptions, run_maintenance -except ImportError: - from maintenance import MaintenanceOptions, run_maintenance +from ..maintenance import MaintenanceOptions, run_maintenance def maintenance_cmd(args) -> int: diff --git a/src/cli/serve.py b/src/cli/serve.py index a8ba095..c2bc1d1 100644 --- a/src/cli/serve.py +++ b/src/cli/serve.py @@ -13,7 +13,7 @@ def serve_cmd(args: Namespace) -> int: port = args.port reload = args.reload - print(f"Starting Open Brain API server...") + print("Starting Open Brain API server...") print(f"Host: {host}") print(f"Port: {port}") print(f"API docs: http://{host}:{port}/docs") @@ -25,7 +25,7 @@ def serve_cmd(args: Namespace) -> int: return 1 uvicorn.run( - "src.api.main:app", + "src.api.main:app" if reload else app, host=host, port=port, reload=reload diff --git a/src/cli/store.py b/src/cli/store.py index 40e67a7..8554957 100644 --- a/src/cli/store.py +++ b/src/cli/store.py @@ -1,7 +1,6 @@ """ Store command for CLI. """ -import json import sys from argparse import Namespace @@ -43,7 +42,7 @@ def store_memory_cmd(args: Namespace) -> int: metadata={} ) - print(f"Memory stored successfully!") + print("Memory stored successfully!") print(f"ID: {memory_id}") print(f"Tags: {', '.join(tags.keys())}") diff --git a/src/connectors/claude_code.py b/src/connectors/claude_code.py index fea3ec1..2dc6bc0 100644 --- a/src/connectors/claude_code.py +++ b/src/connectors/claude_code.py @@ -3,10 +3,8 @@ Imports conversations from Claude Code session logs. """ import json -import os -from datetime import datetime from pathlib import Path -from typing import Dict, Generator, List, Optional +from typing import Dict, List, Optional from ..db.queries import insert_memory from ..embedder import create_embedding diff --git a/src/connectors/file_watcher.py b/src/connectors/file_watcher.py index a2615ca..811aa9f 100644 --- a/src/connectors/file_watcher.py +++ b/src/connectors/file_watcher.py @@ -2,9 +2,7 @@ File watcher connector. Watches a folder for new Markdown files and imports them. """ -import os import time -from datetime import datetime from pathlib import Path from typing import Dict, List, Optional, Callable diff --git a/src/connectors/gmail.py b/src/connectors/gmail.py index 0e930b8..c7252b2 100644 --- a/src/connectors/gmail.py +++ b/src/connectors/gmail.py @@ -2,18 +2,19 @@ Gmail takeout connector. Imports emails from Gmail takeout export. """ +import binascii import json -import os -import zipfile -from datetime import datetime +import logging from pathlib import Path -from typing import Dict, Generator, List, Optional +from typing import Dict, List, Optional from ..db.queries import insert_memory from ..embedder import create_embedding from ..extractors.entities import extract_entities from ..extractors.tagger import auto_tag +logger = logging.getLogger(__name__) + class GmailConnector: """Import emails from Gmail takeout export.""" @@ -166,8 +167,6 @@ def _process_email( date_str = msg.get('date', '') payload = msg.get('payload', {}) - headers = {h['name']: h['value'] for h in payload.get('headers', [])} - # Get body body = '' parts = payload.get('parts', []) @@ -179,8 +178,8 @@ def _process_email( import base64 try: body = base64.urlsafe_b64decode(body).decode('utf-8') - except: - pass + except (binascii.Error, UnicodeDecodeError, ValueError) as exc: + logger.warning("Could not decode Gmail message body: %s", exc) break if not body: diff --git a/src/connectors/telegram.py b/src/connectors/telegram.py index 1cad466..d82fd8f 100644 --- a/src/connectors/telegram.py +++ b/src/connectors/telegram.py @@ -3,10 +3,8 @@ Imports messages from Telegram JSON export. """ import json -import os -from datetime import datetime from pathlib import Path -from typing import Dict, Generator, List, Optional +from typing import Dict, List, Optional from ..db.queries import insert_memory from ..embedder import create_embedding diff --git a/src/connectors/whatsapp.py b/src/connectors/whatsapp.py index 0642628..aca8f29 100644 --- a/src/connectors/whatsapp.py +++ b/src/connectors/whatsapp.py @@ -2,11 +2,10 @@ WhatsApp export connector. Imports messages from WhatsApp chat export. """ -import os import re -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path -from typing import Dict, Generator, List, Optional +from typing import Dict, Optional from ..db.queries import insert_memory from ..embedder import create_embedding @@ -80,12 +79,18 @@ def _parse_message(self, line: str) -> Optional[Dict]: if ',' not in date_str: # Already in simple format pass - parsed_date = datetime.strptime(date_str, '%m/%d/%Y %I:%M %p') + parsed_date = datetime.strptime( + date_str, + '%m/%d/%Y %I:%M %p', + ).replace(tzinfo=datetime.now(timezone.utc).astimezone().tzinfo) except ValueError: try: - parsed_date = datetime.strptime(date_str, '%m/%d/%y %H:%M') + parsed_date = datetime.strptime( + date_str, + '%m/%d/%y %H:%M', + ).replace(tzinfo=datetime.now(timezone.utc).astimezone().tzinfo) except ValueError: - parsed_date = datetime.now() + parsed_date = datetime.now(timezone.utc) return { 'date': parsed_date, diff --git a/src/context/builder.py b/src/context/builder.py index 36cc9e8..b501520 100644 --- a/src/context/builder.py +++ b/src/context/builder.py @@ -5,16 +5,10 @@ from datetime import datetime, timezone from uuid import uuid4 -try: - from .cache import cache_key, load_cached_packet, store_cached_packet - from .models import ContextItem, ContextKind, ContextPacket, ContextRequest, TrustLabel - from ..db.compaction_queries import fetch_active_compactions - from ..db.context_queries import fetch_structured_context, get_scope_revisions -except ImportError: - from context.cache import cache_key, load_cached_packet, store_cached_packet - from context.models import ContextItem, ContextKind, ContextPacket, ContextRequest, TrustLabel - from db.compaction_queries import fetch_active_compactions - from db.context_queries import fetch_structured_context, get_scope_revisions +from .cache import cache_key, load_cached_packet, store_cached_packet +from .models import ContextItem, ContextKind, ContextPacket, ContextRequest, TrustLabel +from ..db.compaction_queries import fetch_active_compactions +from ..db.context_queries import fetch_structured_context, get_scope_revisions _DIVERSITY_ORDER = ( ContextKind.WARNING, ContextKind.NEXT_ACTION, ContextKind.DECISION, diff --git a/src/context/cache.py b/src/context/cache.py index f966998..21ba075 100644 --- a/src/context/cache.py +++ b/src/context/cache.py @@ -8,12 +8,8 @@ from typing import Any from uuid import uuid4 -try: - from .models import ContextPacket, ContextRequest - from ..db.connection import get_db_cursor -except ImportError: # Support legacy execution with src/ directly on sys.path. - from context.models import ContextPacket, ContextRequest - from db.connection import get_db_cursor +from .models import ContextPacket, ContextRequest +from ..db.connection import get_db_cursor CACHE_TTL_SECONDS = 300 MAX_CACHE_ROWS = 5000 diff --git a/src/db/compaction_queries.py b/src/db/compaction_queries.py index 71542ce..5bd609d 100644 --- a/src/db/compaction_queries.py +++ b/src/db/compaction_queries.py @@ -7,22 +7,13 @@ from typing import Any from uuid import UUID -try: - from ..compaction.engine import ( - CompactionCandidate, - POLICY_VERSION, - build_summary, - event_fingerprint, - source_fingerprint, - ) -except ImportError: # Support legacy execution with src/ directly on sys.path. - from compaction.engine import ( - CompactionCandidate, - POLICY_VERSION, - build_summary, - event_fingerprint, - source_fingerprint, - ) +from ..compaction.engine import ( + CompactionCandidate, + POLICY_VERSION, + build_summary, + event_fingerprint, + source_fingerprint, +) from .connection import get_db_cursor diff --git a/src/db/connection.py b/src/db/connection.py index 369d31f..9171f38 100644 --- a/src/db/connection.py +++ b/src/db/connection.py @@ -1,6 +1,7 @@ """ Database connection management for Open Brain. """ +import logging import os import pathlib from contextlib import contextmanager @@ -11,6 +12,8 @@ from psycopg2 import pool from psycopg2.extras import RealDictCursor +logger = logging.getLogger(__name__) + def _detect_local_timezone() -> str: """Detect the system's IANA timezone name; fall back to UTC.""" @@ -19,8 +22,8 @@ def _detect_local_timezone() -> str: parts = str(link).split("zoneinfo/") if len(parts) > 1: return parts[1] - except Exception: - pass + except (OSError, RuntimeError) as exc: + logger.debug("Could not detect local timezone: %s", exc) return "UTC" diff --git a/src/db/consolidation_queries.py b/src/db/consolidation_queries.py index 2e312d4..85cd7c5 100644 --- a/src/db/consolidation_queries.py +++ b/src/db/consolidation_queries.py @@ -8,17 +8,10 @@ from typing import Any from uuid import UUID -try: - from .connection import get_db_cursor -except ImportError: - from db.connection import get_db_cursor - -try: - from ..consolidation.assertions import AssertionCandidate, propose_consolidation - from ..consolidation.execution import validate_execution_contract, validate_reversal_contract -except ImportError: - from consolidation.assertions import AssertionCandidate, propose_consolidation - from consolidation.execution import validate_execution_contract, validate_reversal_contract +from .connection import get_db_cursor + +from ..consolidation.assertions import AssertionCandidate, propose_consolidation +from ..consolidation.execution import validate_execution_contract, validate_reversal_contract POLICY_VERSION = "assertion-consolidation-v1" diff --git a/src/db/context_queries.py b/src/db/context_queries.py index 7cd09f1..98fe1b3 100644 --- a/src/db/context_queries.py +++ b/src/db/context_queries.py @@ -6,10 +6,7 @@ from datetime import datetime, timezone from uuid import UUID -try: - from .connection import get_db_cursor -except ImportError: # Support legacy execution with src/ directly on sys.path. - from db.connection import get_db_cursor +from .connection import get_db_cursor def get_scope_revisions(user_identity_id: UUID | None, project_id: UUID | None, task_id: UUID | None) -> dict[str, int]: diff --git a/src/db/continuity_queries.py b/src/db/continuity_queries.py index 9a509ff..a91fbb3 100644 --- a/src/db/continuity_queries.py +++ b/src/db/continuity_queries.py @@ -5,7 +5,7 @@ import json from typing import Any -from continuity.models import EventCreate, EventRecord, ScopeRef +from ..continuity.models import EventCreate, EventRecord, ScopeRef from .connection import get_db_cursor diff --git a/src/db/import_queries.py b/src/db/import_queries.py index c7abd7d..d6d7da5 100644 --- a/src/db/import_queries.py +++ b/src/db/import_queries.py @@ -6,8 +6,8 @@ from typing import Any from uuid import UUID -from db.connection import get_db_cursor -from importers.base import ImportCandidate +from .connection import get_db_cursor +from ..importers.base import ImportCandidate def create_import_run( diff --git a/src/db/lifecycle_queries.py b/src/db/lifecycle_queries.py index fab4e7b..34a2557 100644 --- a/src/db/lifecycle_queries.py +++ b/src/db/lifecycle_queries.py @@ -7,17 +7,10 @@ from typing import Any from uuid import UUID -try: - from .connection import get_db_cursor -except ImportError: # Support legacy execution with src/ directly on sys.path. - from db.connection import get_db_cursor - -try: - from ..lifecycle.assertions import AssertionLifecycleInput, evaluate_assertion_lifecycle - from ..lifecycle.execution import validate_execution_contract, validate_reversal_contract -except ImportError: # Support legacy execution with src/ directly on sys.path. - from lifecycle.assertions import AssertionLifecycleInput, evaluate_assertion_lifecycle - from lifecycle.execution import validate_execution_contract, validate_reversal_contract +from .connection import get_db_cursor + +from ..lifecycle.assertions import AssertionLifecycleInput, evaluate_assertion_lifecycle +from ..lifecycle.execution import validate_execution_contract, validate_reversal_contract POLICY_VERSION = "assertion-lifecycle-v1" diff --git a/src/db/migrate.py b/src/db/migrate.py index cc1aeb9..cf54bd1 100644 --- a/src/db/migrate.py +++ b/src/db/migrate.py @@ -3,11 +3,43 @@ from __future__ import annotations import hashlib +import re +from collections import defaultdict from importlib.resources import files from src.db.connection import get_db_connection, init_db +_MIGRATION_NAME = re.compile(r"^(?P\d+)_") +_LEGACY_DUPLICATE_SEQUENCES = { + "011": frozenset( + { + "011_assertion_pruning_execution.sql", + "011_context_retrieval_indexes.sql", + } + ), +} + + +def validate_migration_sequence(filenames: list[str]) -> None: + """Reject ambiguous migration numbers except the documented v1.0 legacy pair.""" + grouped: dict[str, set[str]] = defaultdict(set) + for filename in filenames: + match = _MIGRATION_NAME.match(filename) + if not match: + raise RuntimeError(f"invalid migration filename: {filename}") + grouped[match.group("sequence")].add(filename) + + for sequence, names in grouped.items(): + if len(names) < 2: + continue + if _LEGACY_DUPLICATE_SEQUENCES.get(sequence) == frozenset(names): + continue + raise RuntimeError( + f"duplicate migration sequence {sequence}: {', '.join(sorted(names))}" + ) + + def _ensure_ledger(cursor) -> None: cursor.execute( """ @@ -27,6 +59,7 @@ def apply_migrations() -> list[str]: migration_files = sorted( item for item in migration_root.iterdir() if item.name.endswith(".sql") ) + validate_migration_sequence([resource.name for resource in migration_files]) with get_db_connection() as connection: active_migration: str | None = None diff --git a/src/db/pruning_queries.py b/src/db/pruning_queries.py index cc03585..437a36e 100644 --- a/src/db/pruning_queries.py +++ b/src/db/pruning_queries.py @@ -7,17 +7,10 @@ from typing import Any from uuid import UUID -try: - from .connection import get_db_cursor -except ImportError: - from db.connection import get_db_cursor - -try: - from ..pruning.assertions import PruningCandidate, propose_pruning - from ..pruning.execution import validate_archive_contract, validate_restore_contract -except ImportError: - from pruning.assertions import PruningCandidate, propose_pruning - from pruning.execution import validate_archive_contract, validate_restore_contract +from .connection import get_db_cursor + +from ..pruning.assertions import PruningCandidate, propose_pruning +from ..pruning.execution import validate_archive_contract, validate_restore_contract POLICY_VERSION = "assertion-pruning-v1" diff --git a/src/db/queries.py b/src/db/queries.py index c0cef16..ed693e9 100644 --- a/src/db/queries.py +++ b/src/db/queries.py @@ -6,8 +6,6 @@ from datetime import datetime from typing import Any, Dict, List, Optional -from psycopg2 import sql -from psycopg2.extras import RealDictCursor from .connection import get_db_cursor diff --git a/src/db/scope_queries.py b/src/db/scope_queries.py index d778c4c..774d1a1 100644 --- a/src/db/scope_queries.py +++ b/src/db/scope_queries.py @@ -5,8 +5,8 @@ import json from uuid import UUID -from continuity.scopes import IdentityRecord, IdentityRef, SessionOpen, SessionRecord -from db.connection import get_db_cursor +from ..continuity.scopes import IdentityRecord, IdentityRef, SessionOpen, SessionRecord +from .connection import get_db_cursor def _json(value): diff --git a/src/embedder/__init__.py b/src/embedder/__init__.py index c3eaf95..d4e6b1d 100644 --- a/src/embedder/__init__.py +++ b/src/embedder/__init__.py @@ -30,7 +30,7 @@ ``` """ import os -from typing import List, Optional +from typing import ClassVar, List, Optional from abc import ABC, abstractmethod import requests @@ -133,7 +133,7 @@ class OpenRouterEmbedder(BaseEmbedder): """OpenRouter API embedder (default, free tier available).""" # Default free models on OpenRouter - DEFAULT_MODELS = { + DEFAULT_MODELS: ClassVar[dict[str, int]] = { 'text-embedding-3-small': 1536, 'text-embedding-ada-002': 1536, } @@ -216,7 +216,7 @@ def is_available(self) -> bool: class OpenAIEmbedder(BaseEmbedder): """OpenAI API embedder.""" - DEFAULT_MODELS = { + DEFAULT_MODELS: ClassVar[dict[str, int]] = { 'text-embedding-3-small': 1536, 'text-embedding-3-large': 3072, 'text-embedding-ada-002': 1536, diff --git a/src/extractors/entities.py b/src/extractors/entities.py index 1236b2f..132fa5f 100644 --- a/src/extractors/entities.py +++ b/src/extractors/entities.py @@ -3,7 +3,7 @@ Uses basic NLP for named entity recognition. """ import re -from typing import Dict, List, Set, Tuple +from typing import Dict, List, Set import nltk from nltk import pos_tag, word_tokenize diff --git a/src/extractors/tagger.py b/src/extractors/tagger.py index 0a64000..d8d1b57 100644 --- a/src/extractors/tagger.py +++ b/src/extractors/tagger.py @@ -4,7 +4,7 @@ """ import os import re -from typing import Dict, List, Optional, Set +from typing import Dict, List, Optional import yaml diff --git a/src/importers/hermes_context.py b/src/importers/hermes_context.py index c0a793d..904fae7 100644 --- a/src/importers/hermes_context.py +++ b/src/importers/hermes_context.py @@ -4,8 +4,8 @@ from pathlib import Path -from importers.base import ImportCandidate, ImportSource, hash_content, stable_file_id -from importers.hermes_markdown import HermesMarkdownImporter +from .base import ImportCandidate, ImportSource, hash_content, stable_file_id +from .hermes_markdown import HermesMarkdownImporter _ALLOWED_SUFFIXES = {".md", ".markdown", ".txt"} diff --git a/src/importers/hermes_markdown.py b/src/importers/hermes_markdown.py index 51b9e54..a5c3a50 100644 --- a/src/importers/hermes_markdown.py +++ b/src/importers/hermes_markdown.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Iterable -from importers.base import ( +from .base import ( ImportAdapter, ImportCandidate, ImportSource, diff --git a/src/importers/hermes_operations.py b/src/importers/hermes_operations.py index 6713484..930098f 100644 --- a/src/importers/hermes_operations.py +++ b/src/importers/hermes_operations.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Iterable -from importers.base import ImportAdapter, ImportCandidate, ImportSource, hash_content +from .base import ImportAdapter, ImportCandidate, ImportSource, hash_content class HermesSkillImporter(ImportAdapter): diff --git a/src/importers/hermes_sessions.py b/src/importers/hermes_sessions.py index 7912807..4df21fc 100644 --- a/src/importers/hermes_sessions.py +++ b/src/importers/hermes_sessions.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Iterable -from importers.base import ImportAdapter, ImportCandidate, ImportSource, hash_content +from .base import ImportAdapter, ImportCandidate, ImportSource, hash_content class HermesSessionImporter(ImportAdapter): diff --git a/src/importers/providers.py b/src/importers/providers.py index acf6e2f..f0c7fee 100644 --- a/src/importers/providers.py +++ b/src/importers/providers.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field -from importers.base import ImportAdapter, ImportCandidate, ImportSource, hash_content +from .base import ImportAdapter, ImportCandidate, ImportSource, hash_content class ProviderCapability(StrEnum): diff --git a/src/importers/runner.py b/src/importers/runner.py index 5eb9483..ec0a534 100644 --- a/src/importers/runner.py +++ b/src/importers/runner.py @@ -6,14 +6,14 @@ from pydantic import BaseModel, ConfigDict -from db.import_queries import ( +from ..db.import_queries import ( create_import_run, get_import_run, record_import_candidate, seen_external_hashes, update_import_run, ) -from importers.base import ImportAdapter +from .base import ImportAdapter class ImportSummary(BaseModel): @@ -137,4 +137,4 @@ def run_import( records_merged=counters["merged"], records_rejected=counters["rejected"], cursor=cursor, - ) \ No newline at end of file + ) diff --git a/src/importers/staging.py b/src/importers/staging.py index e2ae5a6..64eb9e9 100644 --- a/src/importers/staging.py +++ b/src/importers/staging.py @@ -6,7 +6,7 @@ from typing import Any from uuid import UUID -from db.connection import get_db_cursor +from ..db.connection import get_db_cursor def seal_import_run(run_id: UUID, *, actor: str, expected_records: int | None = None) -> dict[str, Any]: diff --git a/src/ingestion/importer.py b/src/ingestion/importer.py index 5bb2466..2c199fa 100644 --- a/src/ingestion/importer.py +++ b/src/ingestion/importer.py @@ -4,12 +4,10 @@ """ import csv import json -from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Dict from ..db import queries -from ..embedder import create_embeddings from ..extractors.entities import extract_entities from ..extractors.tagger import auto_tag diff --git a/src/main.py b/src/main.py index 76609aa..ee38ad2 100644 --- a/src/main.py +++ b/src/main.py @@ -11,22 +11,18 @@ from mcp.server.stdio import stdio_server from mcp.types import Tool, TextContent -# Add src to path (so `from db import ...` etc. resolve correctly) -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from db import connection -from db.attribution import insert_memory, search_memories -from db.queries import ( - get_memory_by_id, +from .db import connection +from .db.attribution import insert_memory, search_memories +from .db.queries import ( get_related_memories, get_memories_by_entity, get_today_memories, get_memory_stats, ) -from embedder import create_embedding -from extractors.entities import extract_entities -from extractors.tagger import auto_tag -from analytics.weekly_report import generate_weekly_report +from .embedder import create_embedding +from .extractors.entities import extract_entities +from .extractors.tagger import auto_tag +from .analytics.weekly_report import generate_weekly_report def load_config() -> Dict: diff --git a/src/maintenance/orchestrator.py b/src/maintenance/orchestrator.py index 2c1a79f..ec09233 100644 --- a/src/maintenance/orchestrator.py +++ b/src/maintenance/orchestrator.py @@ -8,18 +8,11 @@ from typing import Any, Callable from uuid import UUID, uuid4 -try: - from ..context.cache import cleanup_context_cache, context_cache_stats - from ..db.compaction_queries import compact_events - from ..db.connection import get_db_cursor - from ..db.consolidation_queries import generate_consolidation_proposals - from ..db.pruning_queries import generate_pruning_proposals -except ImportError: - from context.cache import cleanup_context_cache, context_cache_stats - from db.compaction_queries import compact_events - from db.connection import get_db_cursor - from db.consolidation_queries import generate_consolidation_proposals - from db.pruning_queries import generate_pruning_proposals +from ..context.cache import cleanup_context_cache, context_cache_stats +from ..db.compaction_queries import compact_events +from ..db.connection import get_db_cursor +from ..db.consolidation_queries import generate_consolidation_proposals +from ..db.pruning_queries import generate_pruning_proposals @dataclass(frozen=True) diff --git a/src/notifications/email_notifier.py b/src/notifications/email_notifier.py index 663b62b..438c891 100644 --- a/src/notifications/email_notifier.py +++ b/src/notifications/email_notifier.py @@ -5,7 +5,7 @@ import smtplib from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart -from typing import List, Optional +from typing import List class EmailNotifier: diff --git a/src/notifications/telegram_bot.py b/src/notifications/telegram_bot.py index c2dfb34..ea1bdd0 100644 --- a/src/notifications/telegram_bot.py +++ b/src/notifications/telegram_bot.py @@ -2,7 +2,7 @@ Telegram bot notifier for Open Brain. """ import os -from typing import List, Optional +from typing import List import requests @@ -57,7 +57,7 @@ def send_message(self, message: str, parse_mode: str = "Markdown") -> bool: def send_memory_alert(self, memory_content: str, tags: List[str]) -> bool: """Send an alert when important memory is stored.""" - message = f"🧠 *New Memory Stored*\n\n" + message = "🧠 *New Memory Stored*\n\n" message += f"{memory_content[:200]}...\n\n" if tags: message += f"Tags: {', '.join(['#' + t for t in tags])}" @@ -66,7 +66,7 @@ def send_memory_alert(self, memory_content: str, tags: List[str]) -> bool: def send_stats_digest(self, stats: dict) -> bool: """Send daily stats digest.""" - message = f"πŸ“Š *Open Brain Daily Digest*\n\n" + message = "πŸ“Š *Open Brain Daily Digest*\n\n" message += f"Total Memories: {stats.get('total', 0)}\n" by_source = stats.get('by_source', {}) diff --git a/src/openbrain_hermes_plugin/README.md b/src/openbrain_hermes_plugin/README.md index feb6823..f0bbc66 100644 --- a/src/openbrain_hermes_plugin/README.md +++ b/src/openbrain_hermes_plugin/README.md @@ -11,10 +11,11 @@ openbrain install-hermes Then configure Hermes: ```bash -export OPENBRAIN_URL=http://127.0.0.1:8000 hermes memory setup ``` +The OpenBrain installer generates and loads the local URL and API key automatically. Explicit `OPENBRAIN_URL` and `OPENBRAIN_API_KEY` environment variables can override them for remote deployments. + Select `openbrain` as the active provider when prompted, or set `memory.provider: openbrain` in the active Hermes configuration. Optional scope variables: diff --git a/src/openbrain_hermes_plugin/__init__.py b/src/openbrain_hermes_plugin/__init__.py index 8bca876..506a8ee 100644 --- a/src/openbrain_hermes_plugin/__init__.py +++ b/src/openbrain_hermes_plugin/__init__.py @@ -16,23 +16,62 @@ logger = logging.getLogger(__name__) +def _load_openbrain_environment() -> dict[str, str]: + """Load the shared installer configuration without adding dependencies.""" + config_dir = Path( + os.environ.get("OPENBRAIN_CONFIG_DIR", Path.home() / ".config" / "openbrain") + ).expanduser() + values = { + "OPENBRAIN_URL": "http://127.0.0.1:8000", + "OPENBRAIN_TIMEOUT": "3", + } + for path in (config_dir / ".env", Path.cwd() / ".env"): + if not path.is_file(): + continue + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, value = line.split("=", 1) + name = name.removeprefix("export ").strip() + if name.startswith("OPENBRAIN_"): + values[name] = value.strip().strip("\"'") + for name in tuple(values) + ( + "OPENBRAIN_API_KEY", + "OPENBRAIN_PROJECT_ID", + "OPENBRAIN_TASK_ID", + ): + if os.environ.get(name): + values[name] = os.environ[name] + return values + + def _digest(*parts: str) -> str: payload = "\0".join(parts).encode("utf-8") return hashlib.sha256(payload).hexdigest()[:24] class _Client: - def __init__(self, base_url: str, timeout: float = 3.0): + def __init__( + self, + base_url: str, + timeout: float = 3.0, + api_key: str | None = None, + ): self.base_url = base_url.rstrip("/") self.timeout = timeout + self.api_key = api_key def request(self, method: str, path: str, payload: Optional[dict] = None) -> Any: data = json.dumps(payload).encode("utf-8") if payload is not None else None + headers = {"Content-Type": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" request = urllib.request.Request( f"{self.base_url}{path}", data=data, method=method, - headers={"Content-Type": "application/json"}, + headers=headers, ) with urllib.request.urlopen(request, timeout=self.timeout) as response: body = response.read().decode("utf-8") @@ -61,17 +100,22 @@ def name(self) -> str: return "openbrain" def is_available(self) -> bool: - return bool(os.environ.get("OPENBRAIN_URL", "http://127.0.0.1:8000").strip()) + return bool(_load_openbrain_environment()["OPENBRAIN_URL"].strip()) def initialize(self, session_id: str, **kwargs) -> None: self._session_id = session_id self._platform = str(kwargs.get("platform") or "cli") self._hermes_home = Path(kwargs.get("hermes_home") or Path.home() / ".hermes") self._spool = self._hermes_home / "openbrain-spool.jsonl" - self._project_id = os.environ.get("OPENBRAIN_PROJECT_ID") or None - self._task_id = os.environ.get("OPENBRAIN_TASK_ID") or None - timeout = float(os.environ.get("OPENBRAIN_TIMEOUT", "3")) - self._client = _Client(os.environ.get("OPENBRAIN_URL", "http://127.0.0.1:8000"), timeout) + settings = _load_openbrain_environment() + self._project_id = settings.get("OPENBRAIN_PROJECT_ID") or None + self._task_id = settings.get("OPENBRAIN_TASK_ID") or None + timeout = float(settings.get("OPENBRAIN_TIMEOUT", "3")) + self._client = _Client( + settings["OPENBRAIN_URL"], + timeout, + settings.get("OPENBRAIN_API_KEY"), + ) user_key = str(kwargs.get("user_id") or kwargs.get("user_id_alt") or "default-user") agent_key = str(kwargs.get("agent_identity") or "hermes") diff --git a/src/openbrain_hermes_plugin/plugin.yaml b/src/openbrain_hermes_plugin/plugin.yaml index 1a9dec3..4e8f548 100644 --- a/src/openbrain_hermes_plugin/plugin.yaml +++ b/src/openbrain_hermes_plugin/plugin.yaml @@ -1,5 +1,5 @@ name: openbrain -version: 0.2.0 +version: 1.0.0 description: Shared personal memory and cross-agent continuity through Open Brain hooks: - on_session_end diff --git a/src/providers/host_config.py b/src/providers/host_config.py index 7d298ad..e60eaee 100644 --- a/src/providers/host_config.py +++ b/src/providers/host_config.py @@ -6,6 +6,8 @@ from dataclasses import dataclass from pathlib import Path +from src.runtime_config import load_runtime_environment, restrict_file_permissions + @dataclass(frozen=True) class HostAdapterConfig: @@ -17,6 +19,7 @@ class HostAdapterConfig: @classmethod def from_env(cls) -> "HostAdapterConfig": + load_runtime_environment() base_url = os.getenv("OPENBRAIN_URL", "http://127.0.0.1:8000").strip().rstrip("/") api_key = os.getenv("OPENBRAIN_API_KEY") or None try: @@ -59,7 +62,7 @@ def install_env_file(path: Path, config: HostAdapterConfig, *, overwrite: bool = path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text(render_env_file(config), encoding="utf-8") - temporary.chmod(0o600) + restrict_file_permissions(temporary) temporary.replace(path) return path diff --git a/src/runtime_config.py b/src/runtime_config.py new file mode 100644 index 0000000..8997474 --- /dev/null +++ b/src/runtime_config.py @@ -0,0 +1,181 @@ +"""Shared, private runtime configuration for the server and its clients.""" + +from __future__ import annotations + +import csv +import os +import secrets +import shutil +import subprocess +from pathlib import Path + +from dotenv import dotenv_values, load_dotenv, set_key + +_RUNTIME_KEYS = ( + "OPENBRAIN_URL", + "OPENBRAIN_API_KEY", + "OPENBRAIN_AUTH_REQUIRED", + "OPENBRAIN_TIMEOUT", +) +_PLACEHOLDERS = { + "", + "change-me", + "openbrain", + "replace-me", + "replace-with-generated-secret", + "your_secure_database_password", + "your_openbrain_api_key", +} + + +def runtime_config_dir() -> Path: + configured = os.getenv("OPENBRAIN_CONFIG_DIR") + if configured: + return Path(configured).expanduser() + return Path.home() / ".config" / "openbrain" + + +def runtime_env_path() -> Path: + return runtime_config_dir() / ".env" + + +def restrict_file_permissions(path: Path) -> None: + """Restrict a secret-bearing file to the current user on each platform.""" + if os.name != "nt": + path.chmod(0o600) + return + + identity = subprocess.run( + ["whoami", "/user", "/fo", "csv", "/nh"], + check=True, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ).stdout + user_sid = next(csv.reader([identity]))[1] + subprocess.run( + [ + "icacls", + str(path), + "/inheritance:r", + "/grant:r", + f"*{user_sid}:(F)", + "/grant:r", + "*S-1-5-18:(F)", + "/grant:r", + "*S-1-5-32-544:(F)", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def _usable_secret(value: str | None) -> bool: + return bool(value and len(value) >= 32 and value.lower() not in _PLACEHOLDERS) + + +def _read_key(path: Path) -> str | None: + if not path.is_file(): + return None + value = dotenv_values(path).get("OPENBRAIN_API_KEY") + return str(value) if value else None + + +def _ensure_env_file( + path: Path, + api_key: str, + *, + include_database: bool = False, +) -> Path: + path = path.expanduser().resolve() + path.parent.mkdir(parents=True, exist_ok=True) + if not path.exists(): + path.write_text("# Open Brain private runtime configuration\n", encoding="utf-8") + + current = dotenv_values(path) + defaults = { + "OPENBRAIN_URL": "http://127.0.0.1:8000", + "OPENBRAIN_API_KEY": api_key, + "OPENBRAIN_AUTH_REQUIRED": "true", + "OPENBRAIN_TIMEOUT": "3", + } + for name, value in defaults.items(): + existing = current.get(name) + if name == "OPENBRAIN_API_KEY" or not existing: + set_key(str(path), name, value, quote_mode="never") + if include_database: + database_password = current.get("DB_PASSWORD") + if not database_password or str(database_password).lower() in _PLACEHOLDERS: + set_key( + str(path), + "DB_PASSWORD", + secrets.token_urlsafe(24), + quote_mode="never", + ) + restrict_file_permissions(path) + return path + + +def configure_runtime_environment(project_root: Path | None = None) -> Path: + """Generate credentials once and share them with local server and clients.""" + user_path = runtime_env_path() + project_path = ( + project_root.expanduser().resolve() / ".env" if project_root is not None else None + ) + if project_path and not project_path.exists(): + example = project_path.parent / ".env.example" + if example.is_file(): + project_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(example, project_path) + candidates = ( + os.getenv("OPENBRAIN_API_KEY"), + _read_key(project_path) if project_path else None, + _read_key(user_path), + ) + api_key = next((value for value in candidates if _usable_secret(value)), None) + if api_key is None: + api_key = secrets.token_urlsafe(32) + + _ensure_env_file(user_path, api_key) + target = ( + _ensure_env_file(project_path, api_key, include_database=True) + if project_path + else user_path + ) + os.environ["OPENBRAIN_API_KEY"] = api_key + os.environ.setdefault("OPENBRAIN_URL", "http://127.0.0.1:8000") + os.environ.setdefault("OPENBRAIN_AUTH_REQUIRED", "true") + os.environ.setdefault("OPENBRAIN_TIMEOUT", "3") + return target + + +def load_runtime_environment() -> None: + """Load project-local settings first, then the per-user installation settings.""" + candidates: list[Path] = [] + explicit = os.getenv("OPENBRAIN_ENV_FILE") + if explicit: + candidates.append(Path(explicit).expanduser()) + candidates.append(Path.cwd() / ".env") + source_root = Path(__file__).resolve().parents[1] + candidates.append(source_root / ".env") + candidates.append(runtime_env_path()) + + seen: set[Path] = set() + for candidate in candidates: + resolved = candidate.resolve() + if resolved in seen or not resolved.is_file(): + continue + seen.add(resolved) + load_dotenv(resolved, override=False) + + +def runtime_settings_from_file(path: Path | None = None) -> dict[str, str]: + """Return only Open Brain runtime values without exposing unrelated secrets.""" + values = dotenv_values(path or runtime_env_path()) + return { + key: str(values[key]) + for key in _RUNTIME_KEYS + if values.get(key) is not None + } diff --git a/src/sandbox/__init__.py b/src/sandbox/__init__.py index 28762b3..e48e9e1 100644 --- a/src/sandbox/__init__.py +++ b/src/sandbox/__init__.py @@ -13,7 +13,7 @@ import os import asyncio -from datetime import timedelta +import shlex from typing import Optional, Dict, Any, List from dataclasses import dataclass @@ -166,7 +166,7 @@ async def run_python( Returns: ExecutionResult with stdout, stderr """ - return await self.run(f"python3 -c '{code.replace(\"'\", \"'\\\")\")}'", timeout=timeout) + return await self.run(f"python3 -c {shlex.quote(code)}", timeout=timeout) class DirectExecutor: diff --git a/src/sandbox/cli.py b/src/sandbox/cli.py index c9a7b38..8c7d668 100644 --- a/src/sandbox/cli.py +++ b/src/sandbox/cli.py @@ -53,11 +53,11 @@ async def run_command(args): print(f" Error: {result.error}") if result.stdout: - print(f"\n--- stdout ---") + print("\n--- stdout ---") print(result.stdout[:2000] if len(result.stdout) > 2000 else result.stdout) if result.stderr: - print(f"\n--- stderr ---") + print("\n--- stderr ---") print(result.stderr[:2000] if len(result.stderr) > 2000 else result.stderr) return 0 if result.success else 1 diff --git a/src/sandbox/docker_sandbox.py b/src/sandbox/docker_sandbox.py index 8722ebf..8b56164 100644 --- a/src/sandbox/docker_sandbox.py +++ b/src/sandbox/docker_sandbox.py @@ -12,13 +12,15 @@ """ import asyncio -import json +import logging +import shlex import uuid -from datetime import timedelta -from typing import Optional, Dict, Any, List +from typing import Optional, Dict, List from dataclasses import dataclass import subprocess +logger = logging.getLogger(__name__) + @dataclass class ExecutionResult: @@ -63,16 +65,18 @@ def __init__( self.cpu_limit = cpu_limit self._container_id: Optional[str] = None - def _ensure_image(self) -> bool: + async def _ensure_image(self) -> bool: """Pull image if needed.""" try: - result = subprocess.run( + result = await asyncio.to_thread( + subprocess.run, ["docker", "pull", self.image], capture_output=True, - timeout=60 + timeout=60, + check=False, ) return result.returncode == 0 - except Exception: + except (OSError, subprocess.SubprocessError): return False async def run( @@ -100,7 +104,7 @@ async def run( timeout = timeout or self.timeout # Ensure image exists - if not self._ensure_image(): + if not await self._ensure_image(): return ExecutionResult( success=False, stdout="", @@ -138,11 +142,13 @@ async def run( cmd.extend([self.image, "sh", "-c", command]) try: - result = subprocess.run( + result = await asyncio.to_thread( + subprocess.run, cmd, capture_output=True, text=True, - timeout=timeout + timeout=timeout, + check=False, ) duration_ms = int((time.time() - start) * 1000) @@ -157,8 +163,12 @@ async def run( except subprocess.TimeoutExpired: # Kill the container if still running - subprocess.run(["docker", "kill", container_name], - capture_output=True) + await asyncio.to_thread( + subprocess.run, + ["docker", "kill", container_name], + capture_output=True, + check=False, + ) return ExecutionResult( success=False, stdout="", @@ -183,7 +193,7 @@ async def run_python( timeout: Optional[int] = None ) -> ExecutionResult: """Run Python code in the sandbox.""" - return await self.run(f"python3 -c '{code.replace(\"'\", \"'\\\\\")\")}'", timeout=timeout) + return await self.run(f"python3 -c {shlex.quote(code)}", timeout=timeout) def is_available(self) -> bool: """Check if Docker is available.""" @@ -191,7 +201,8 @@ def is_available(self) -> bool: result = subprocess.run( ["docker", "version"], capture_output=True, - timeout=5 + timeout=5, + check=False, ) return result.returncode == 0 except Exception: @@ -221,20 +232,36 @@ async def initialize(self): for i in range(self.pool_size): container_name = f"sandbox-pool-{i}-{uuid.uuid4().hex[:4]}" try: - subprocess.run([ - "docker", "run", "-d", - "--name", container_name, - "--network", "none", - "--memory", "512m", - "--cpus", "1.0", - "--read-only", - "--tmpfs", "/tmp:rw,size=64m", - self.image, - "sleep", "infinity" - ], capture_output=True, timeout=30) - self._containers.append(container_name) - except Exception: - pass + result = await asyncio.to_thread( + subprocess.run, + [ + "docker", "run", "-d", + "--name", container_name, + "--network", "none", + "--memory", "512m", + "--cpus", "1.0", + "--read-only", + "--tmpfs", "/tmp:rw,size=64m", + self.image, + "sleep", "infinity", + ], + capture_output=True, + timeout=30, + check=False, + ) + if result.returncode == 0: + self._containers.append(container_name) + else: + logger.warning( + "Could not start sandbox pool container %s", + container_name, + ) + except (OSError, subprocess.SubprocessError) as exc: + logger.warning( + "Could not start sandbox pool container %s: %s", + container_name, + exc, + ) async def execute(self, command: str) -> ExecutionResult: """Execute command in a pooled container.""" @@ -247,10 +274,14 @@ async def execute(self, command: str) -> ExecutionResult: container = self._containers[0] try: - result = subprocess.run([ - "docker", "exec", container, - "sh", "-c", command - ], capture_output=True, text=True, timeout=self.timeout) + result = await asyncio.to_thread( + subprocess.run, + ["docker", "exec", container, "sh", "-c", command], + capture_output=True, + text=True, + timeout=self.timeout, + check=False, + ) return ExecutionResult( success=result.returncode == 0, @@ -272,8 +303,18 @@ async def execute(self, command: str) -> ExecutionResult: async def cleanup(self): """Stop and remove all pooled containers.""" for container in self._containers: - subprocess.run(["docker", "kill", container], capture_output=True) - subprocess.run(["docker", "rm", container], capture_output=True) + await asyncio.to_thread( + subprocess.run, + ["docker", "kill", container], + capture_output=True, + check=False, + ) + await asyncio.to_thread( + subprocess.run, + ["docker", "rm", container], + capture_output=True, + check=False, + ) self._containers = [] diff --git a/src/setup/__init__.py b/src/setup/__init__.py index ee1136f..1d90eeb 100644 --- a/src/setup/__init__.py +++ b/src/setup/__init__.py @@ -2,10 +2,117 @@ Open Brain Setup Wizard - Interactive first-run configuration """ +import copy import os +import secrets import yaml from pathlib import Path -from typing import Dict, Any +from typing import Any, Dict, Mapping + +from ..providers.host_config import restrict_file_permissions +from ..runtime_config import configure_runtime_environment + + +_ENVIRONMENT_KEYS = ( + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "MINIMAX_API_KEY", + "CUSTOM_API_KEY", + "OLLAMA_BASE_URL", + "OPENBRAIN_API_KEY", +) +_SECRET_CONFIG_KEYS = {"password", "api_key", "token", "bot_token", "smtp_password"} + + +def _without_secrets(value: Any) -> Any: + if isinstance(value, dict): + return { + key: _without_secrets(item) + for key, item in value.items() + if key.lower() not in _SECRET_CONFIG_KEYS + } + if isinstance(value, list): + return [_without_secrets(item) for item in value] + return copy.deepcopy(value) + + +def _render_env_file( + config: Mapping[str, Any], + environment: Mapping[str, str], +) -> str: + database = config.get("database", {}) + api = config.get("api", {}) + dashboard = config.get("dashboard", {}) + security = config.get("security", {}) + embedder = config.get("embedder", {}) + + values: dict[str, Any] = { + "DB_HOST": database.get("host", "localhost"), + "DB_PORT": database.get("port", 5432), + "DB_NAME": database.get("name", "openbrain"), + "DB_USER": database.get("user", "postgres"), + "DB_PASSWORD": database.get("password", ""), + "DB_TIMEZONE": database.get("timezone", "auto"), + "EMBEDDER_PROVIDER": embedder.get("provider", "ollama"), + "EMBED_MODEL": embedder.get("model", ""), + "API_PORT": api.get("port", 8000), + "DASHBOARD_PORT": dashboard.get("port", 8501), + "SECURITY_MODE": security.get("mode", "direct"), + } + if embedder.get("ollama_base_url"): + values["OLLAMA_BASE_URL"] = embedder["ollama_base_url"] + if embedder.get("custom_base_url"): + values["CUSTOM_API_URL"] = embedder["custom_base_url"] + + notifications = config.get("analytics", {}).get("notifications", {}) + telegram = notifications.get("telegram", {}) + email = notifications.get("email", {}) + values.update( + { + "TELEGRAM_BOT_TOKEN": telegram.get("bot_token", ""), + "TELEGRAM_CHAT_ID": telegram.get("chat_id", ""), + "SMTP_HOST": email.get("smtp_host", ""), + "SMTP_PORT": email.get("smtp_port", ""), + "SMTP_USER": email.get("smtp_user", ""), + "SMTP_PASSWORD": email.get("smtp_password", ""), + "EMAIL_FROM": email.get("from_email", ""), + } + ) + for key in _ENVIRONMENT_KEYS: + if environment.get(key): + values[key] = environment[key] + values.setdefault("OPENBRAIN_API_KEY", secrets.token_urlsafe(32)) + values.setdefault("OPENBRAIN_AUTH_REQUIRED", "true") + values.setdefault("OPENBRAIN_URL", "http://127.0.0.1:8000") + + lines = ["# Open Brain environment generated by the setup wizard"] + lines.extend(f"{key}={value}" for key, value in values.items() if value != "") + return "\n".join(lines) + "\n" + + +def save_configuration( + config: Mapping[str, Any], + *, + project_root: Path, + environment: Mapping[str, str] | None = None, +) -> tuple[Path, Path]: + """Write non-secret YAML configuration and a private environment file.""" + project_root = project_root.expanduser().resolve() + config_path = project_root / "config" / "settings.yaml" + env_path = project_root / ".env" + config_path.parent.mkdir(parents=True, exist_ok=True) + + config_path.write_text( + yaml.safe_dump(_without_secrets(config), default_flow_style=False, sort_keys=False), + encoding="utf-8", + ) + env_path.write_text( + _render_env_file(config, environment or os.environ), + encoding="utf-8", + ) + restrict_file_permissions(env_path) + return config_path, env_path def prompt_env(var_name: str, default: str = "", secret: bool = False) -> str: @@ -38,7 +145,7 @@ def run_setup() -> Dict[str, Any]: 'port': int(prompt_env("DB_PORT", "5432")), 'name': prompt_env("DB_NAME", "openbrain"), 'user': prompt_env("DB_USER", "postgres"), - 'password': prompt_env("DB_PASSWORD", "", secret=True) or "openbrain", + 'password': prompt_env("DB_PASSWORD", "", secret=True) or secrets.token_urlsafe(24), } # ═══════════════════════════════════════════════════════ @@ -47,7 +154,7 @@ def run_setup() -> Dict[str, Any]: print("\n🧠 EMBEDDING PROVIDER") print("─" * 50) print("Choose your embedding provider:") - print(" 1. OpenRouter - FREE, no setup (recommended)") + print(" 1. OpenRouter - Hosted embeddings (API key required)") print(" 2. OpenAI - GPT embeddings") print(" 3. Ollama - Local embeddings") print(" 4. Custom - Any OpenAI-compatible API") @@ -95,7 +202,7 @@ def run_setup() -> Dict[str, Any]: print("\nπŸ€– LLM PROVIDER (for MCP)") print("─" * 50) print("Choose LLM for the MCP server:") - print(" 1. OpenRouter - FREE tier available") + print(" 1. OpenRouter - Hosted models (API key required)") print(" 2. OpenAI - GPT-4, GPT-4o") print(" 3. Anthropic - Claude models") print(" 4. Ollama - Local models") @@ -146,14 +253,13 @@ def run_setup() -> Dict[str, Any]: print("\n🌐 API SERVER") print("─" * 50) config['api'] = { - 'host': '0.0.0.0', + 'host': '127.0.0.1', 'port': int(prompt_env("API_PORT", "8000")), - 'cors_origins': ['*'], + 'cors_origins': ['http://localhost:8501', 'http://localhost:8000'], } config['mcp'] = { - 'host': '0.0.0.0', - 'port': int(prompt_env("MCP_PORT", "8080")), + 'transport': 'stdio', } config['dashboard'] = { @@ -208,37 +314,15 @@ def run_setup() -> Dict[str, Any]: print("\nπŸ’Ύ SAVING CONFIGURATION") print("─" * 50) - # Save to config/settings.yaml - config_path = Path(__file__).parent.parent / "config" / "settings.yaml" - config_path.parent.mkdir(parents=True, exist_ok=True) - - with open(config_path, 'w') as f: - yaml.dump(config, f, default_flow_style=False, sort_keys=False) - + project_root = Path(__file__).resolve().parents[2] + config_path, env_path = save_configuration( + config, + project_root=project_root, + environment=os.environ, + ) + configure_runtime_environment(project_root=project_root) print(f"βœ“ Configuration saved to: {config_path}") - - # Save .env file - env_path = Path(__file__).parent.parent / ".env" - env_vars = [] - for key in ['OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', - 'MINIMAX_API_KEY', 'CUSTOM_API_KEY', 'OLLAMA_BASE_URL', - 'TELEGRAM_BOT_TOKEN', 'TELEGRAM_CHAT_ID', 'SMTP_HOST', - 'SMTP_PORT', 'SMTP_USER', 'SMTP_PASSWORD', 'EMAIL_FROM']: - value = os.environ.get(key) - if value: - env_vars.append(f"{key}={value}") - - if env_vars: - with open(env_path, 'w') as f: - f.write("# Open Brain Environment Variables\n") - f.write("# Copy this to .env and fill in remaining values\n\n") - for key in ['DB_HOST', 'DB_PORT', 'DB_NAME', 'DB_USER', 'DB_PASSWORD', - 'API_PORT', 'MCP_PORT', 'DASHBOARD_PORT', 'SECURITY_MODE']: - f.write(f"{key}={config.get(key.split('_')[0].lower(), {}).get(key, '') if key in ['API_PORT', 'MCP_PORT', 'DASHBOARD_PORT'] else ''}\n") - f.write("\n# API Keys\n") - f.write("\n".join(env_vars)) - - print(f"βœ“ Environment variables saved to: {env_path}") + print(f"βœ“ Environment variables saved to: {env_path}") print(""" ╔═══════════════════════════════════════════════════════════════╗ diff --git a/src/version.py b/src/version.py new file mode 100644 index 0000000..e42533f --- /dev/null +++ b/src/version.py @@ -0,0 +1,22 @@ +"""Resolve the Open Brain release version from source or installed metadata.""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +import tomllib + + +def get_version() -> str: + """Return the project version in both source checkouts and installed wheels.""" + pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml" + if pyproject.is_file(): + project = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("project", {}) + source_version = project.get("version") + if isinstance(source_version, str) and source_version: + return source_version + + try: + return version("openbrain") + except PackageNotFoundError: + return "development" diff --git a/tests/test_agent_attribution.py b/tests/test_agent_attribution.py index 3ee398f..ddb9af1 100644 --- a/tests/test_agent_attribution.py +++ b/tests/test_agent_attribution.py @@ -1,13 +1,8 @@ """Tests for exact multi-agent memory attribution.""" -import os -import sys from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timezone from unittest.mock import Mock, patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) - - def _cursor(rows=None, row=None): cursor = Mock() cursor.fetchall.return_value = rows or [] @@ -21,7 +16,7 @@ def manager(): def test_insert_memory_persists_captured_by(): - from db import attribution + from src.db import attribution cursor, manager = _cursor() with patch.object(attribution, "get_db_cursor", manager): @@ -37,7 +32,7 @@ def test_insert_memory_persists_captured_by(): def test_insert_memory_allows_legacy_null_attribution(): - from db import attribution + from src.db import attribution cursor, manager = _cursor() with patch.object(attribution, "get_db_cursor", manager): @@ -48,7 +43,7 @@ def test_insert_memory_allows_legacy_null_attribution(): def test_semantic_search_filters_by_exact_agent(): - from db import attribution + from src.db import attribution cursor, manager = _cursor() with patch.object(attribution, "get_db_cursor", manager): @@ -65,7 +60,7 @@ def test_semantic_search_filters_by_exact_agent(): def test_text_search_combines_transport_and_agent_filters(): - from db import attribution + from src.db import attribution cursor, manager = _cursor() with patch.object(attribution, "get_db_cursor", manager): @@ -87,9 +82,9 @@ def test_text_search_combines_transport_and_agent_filters(): def test_recent_memories_filter_by_agent_and_decode_rows(): - from db import attribution + from src.db import attribution - now = datetime.now() + now = datetime.now(timezone.utc) cursor, manager = _cursor(rows=[{ "id": "memory-1", "source": "mcp", @@ -119,7 +114,7 @@ def test_recent_memories_filter_by_agent_and_decode_rows(): def test_rest_models_expose_agent_attribution(): - from api.main import MemoryCreate, MemoryResponse, SearchRequest + from src.api.main import MemoryCreate, MemoryResponse, SearchRequest assert "captured_by" in MemoryCreate.model_fields assert "captured_by" in MemoryResponse.model_fields @@ -128,7 +123,7 @@ def test_rest_models_expose_agent_attribution(): def test_mcp_tool_schemas_expose_agent_attribution(): import asyncio - import main + from src import main tools = asyncio.run(main.list_tools()) schemas = {tool.name: tool.inputSchema for tool in tools} @@ -137,7 +132,7 @@ def test_mcp_tool_schemas_expose_agent_attribution(): def test_mcp_format_includes_attribution_when_present(): - from main import format_memory_list + from src.main import format_memory_list formatted = format_memory_list([{ "id": "memory-1", @@ -145,6 +140,6 @@ def test_mcp_format_includes_attribution_when_present(): "captured_by": "medusa:session-9", "content": "Completed task", "tags": [], - "created_at": datetime.now(), + "created_at": datetime.now(timezone.utc), }]) assert "Captured by: medusa:session-9" in formatted diff --git a/tests/test_code_quality_regressions.py b/tests/test_code_quality_regressions.py new file mode 100644 index 0000000..808fc0d --- /dev/null +++ b/tests/test_code_quality_regressions.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import asyncio +import json +import os +import py_compile +import re +import shlex +import subprocess +import sys +from pathlib import Path +import tomllib + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def project_version() -> str: + pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + return pyproject["project"]["version"] + + +def test_shipped_sandbox_modules_compile() -> None: + for relative_path in ("src/sandbox/__init__.py", "src/sandbox/docker_sandbox.py"): + py_compile.compile(str(ROOT / relative_path), doraise=True) + + +@pytest.mark.parametrize( + ("module_name", "class_name"), + ( + ("src.sandbox", "SandboxExecutor"), + ("src.sandbox.docker_sandbox", "DockerSandbox"), + ), +) +def test_sandbox_python_runner_preserves_arbitrary_code( + module_name: str, + class_name: str, +) -> None: + module = __import__(module_name, fromlist=[class_name]) + sandbox = getattr(module, class_name)() + captured: dict[str, object] = {} + + async def capture(command: str, timeout: int | None = None): + captured["command"] = command + captured["timeout"] = timeout + return None + + sandbox.run = capture + code = "print(\"quotes: 'single' and \\\\ backslash\")" + asyncio.run(sandbox.run_python(code, timeout=17)) + + assert shlex.split(str(captured["command"])) == ["python3", "-c", code] + assert captured["timeout"] == 17 + + +def test_api_uses_distribution_version_without_duplicate_namespace() -> None: + script = """ +import json +import sys +from src.api.main import app +print(json.dumps({ + "version": app.version, + "duplicate_api_namespace": any( + name == "api" or name.startswith("api.") for name in sys.modules + ), +})) +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout) + + assert payload["version"] == project_version() + assert payload["duplicate_api_namespace"] is False + + +def test_setup_saves_complete_env_in_project_root_without_yaml_secrets( + tmp_path: Path, +) -> None: + from src.setup import save_configuration + + config = { + "database": { + "host": "database.internal", + "port": 5432, + "name": "openbrain", + "user": "brain", + "password": "database-secret", + }, + "embedder": { + "provider": "openai", + "model": "text-embedding-3-small", + "dimensions": 1536, + }, + "llm": {"provider": "ollama", "model": "llama3"}, + "api": {"host": "127.0.0.1", "port": 8000, "cors_origins": []}, + "mcp": {"transport": "stdio"}, + "dashboard": {"port": 8501}, + "security": {"mode": "direct"}, + "analytics": { + "notifications": { + "telegram": {"enabled": True, "bot_token": "telegram-secret"}, + }, + }, + } + + config_path, env_path = save_configuration( + config, + project_root=tmp_path, + environment={"OPENAI_API_KEY": "provider-secret"}, + ) + + assert config_path == tmp_path / "config" / "settings.yaml" + assert env_path == tmp_path / ".env" + + env_text = env_path.read_text(encoding="utf-8") + assert "DB_HOST=database.internal" in env_text + assert "DB_PORT=5432" in env_text + assert "DB_NAME=openbrain" in env_text + assert "DB_USER=brain" in env_text + assert "DB_PASSWORD=database-secret" in env_text + assert "API_PORT=8000" in env_text + assert "DASHBOARD_PORT=8501" in env_text + assert "SECURITY_MODE=direct" in env_text + assert "OPENAI_API_KEY=provider-secret" in env_text + generated_key = re.search(r"^OPENBRAIN_API_KEY=(.+)$", env_text, re.MULTILINE) + assert generated_key + assert len(generated_key.group(1)) >= 32 + assert "MCP_PORT" not in env_text + + saved_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + assert "password" not in saved_config["database"] + assert "bot_token" not in saved_config["analytics"]["notifications"]["telegram"] + + +def test_install_configuration_generates_and_reuses_transparent_credentials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.runtime_config import configure_runtime_environment + + config_home = tmp_path / "user-config" + project_root = tmp_path / "project" + project_root.mkdir() + (project_root / ".env.example").write_text( + "DB_PASSWORD=openbrain\n", + encoding="utf-8", + ) + monkeypatch.setenv("OPENBRAIN_CONFIG_DIR", str(config_home)) + monkeypatch.delenv("OPENBRAIN_API_KEY", raising=False) + + project_env = configure_runtime_environment(project_root=project_root) + first_key = re.search( + r"^OPENBRAIN_API_KEY=(.+)$", + project_env.read_text(encoding="utf-8"), + re.MULTILINE, + ) + assert first_key + assert len(first_key.group(1)) >= 32 + assert "OPENBRAIN_AUTH_REQUIRED=true" in project_env.read_text(encoding="utf-8") + generated_db_password = re.search( + r"^DB_PASSWORD=(.+)$", + project_env.read_text(encoding="utf-8"), + re.MULTILINE, + ) + assert generated_db_password + assert len(generated_db_password.group(1)) >= 24 + + user_env = config_home / ".env" + assert user_env.is_file() + assert f"OPENBRAIN_API_KEY={first_key.group(1)}" in user_env.read_text(encoding="utf-8") + + monkeypatch.delenv("OPENBRAIN_API_KEY", raising=False) + configure_runtime_environment(project_root=project_root) + second_key = re.search( + r"^OPENBRAIN_API_KEY=(.+)$", + project_env.read_text(encoding="utf-8"), + re.MULTILINE, + ) + assert second_key + assert second_key.group(1) == first_key.group(1) + + if os.name != "nt": + assert oct(project_env.stat().st_mode & 0o777) == "0o600" + assert oct(user_env.stat().st_mode & 0o777) == "0o600" + + +def test_compose_requires_database_secret_and_binds_postgres_locally() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + + assert "DB_PASSWORD: openbrain" not in compose + assert "POSTGRES_PASSWORD: openbrain" not in compose + assert compose.count("${DB_PASSWORD:?") == 3 + assert '127.0.0.1:${DB_HOST_PORT:-5433}:5432' in compose + + +def test_migration_sequence_rejects_new_duplicate_numbers() -> None: + from src.db.migrate import validate_migration_sequence + + current = sorted(path.name for path in (ROOT / "src/db/migrations").glob("*.sql")) + validate_migration_sequence(current) + + with pytest.raises(RuntimeError, match="duplicate migration sequence 099"): + validate_migration_sequence(["099_first.sql", "099_second.sql"]) + + +def test_runtime_and_hermes_plugin_versions_match_distribution() -> None: + import src + + plugin = yaml.safe_load( + (ROOT / "src/openbrain_hermes_plugin/plugin.yaml").read_text(encoding="utf-8") + ) + assert src.__version__ == project_version() + assert plugin["version"] == project_version() + + +def test_generated_distribution_metadata_is_not_tracked_as_source() -> None: + tracked = subprocess.run( + ["git", "ls-files", "openbrain.egg-info"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout + assert not tracked.strip() + + +def test_ci_compiles_all_modules_and_runs_correctness_lint() -> None: + workflow = (ROOT / ".github/workflows/verify.yml").read_text(encoding="utf-8") + + assert "python -m compileall -q src scripts tests" in workflow + assert ( + "ruff check src scripts tests --select " + "E9,F401,F541,F63,F7,F82,E722,F841,S110,ASYNC221," + "PLW1508,PLW1510,RUF012,B017,DTZ001,DTZ005,DTZ007" + ) in workflow + assert "python -m src.cli configure --project-root ." in workflow + assert "docker compose config --quiet" in workflow diff --git a/tests/test_continuity_models.py b/tests/test_continuity_models.py index 74ff04e..b9fa8b8 100644 --- a/tests/test_continuity_models.py +++ b/tests/test_continuity_models.py @@ -24,7 +24,7 @@ def test_event_requires_timezone_aware_timestamp(): idempotency_key="hermes:test:0002", source_system="hermes", payload={"content": "hello"}, - occurred_at=datetime(2026, 7, 21, 12, 0, 0), + occurred_at=datetime(2026, 7, 21, 12, 0, 0), # noqa: DTZ001 - intentionally naive ) diff --git a/tests/test_core.py b/tests/test_core.py index dd89c81..7425e35 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2,17 +2,12 @@ Core tests for Open Brain. """ import os -import sys import uuid -from datetime import datetime +from datetime import datetime, timezone from unittest.mock import Mock, patch import pytest -# Add src to path for legacy top-level imports exercised by the existing suite. -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - - class TestConfig: """Test configuration loading.""" @@ -38,26 +33,26 @@ class TestEntityExtractor: """Test entity extraction.""" def test_email_extraction(self): - from extractors.entities import extract_entities + from src.extractors.entities import extract_entities entities = extract_entities("Contact me at test@example.com") assert 'test@example.com' in entities['emails'] def test_url_extraction(self): - from extractors.entities import extract_entities + from src.extractors.entities import extract_entities entities = extract_entities("Check out https://github.com/test/repo") assert any('github.com' in url for url in entities['urls']) def test_hashtag_extraction(self): - from extractors.entities import extract_entities + from src.extractors.entities import extract_entities entities = extract_entities("Great #python #ai project") assert '#python' in entities['hashtags'] assert '#ai' in entities['hashtags'] def test_technology_extraction(self): - from extractors.entities import extract_entities + from src.extractors.entities import extract_entities entities = extract_entities("Built with Python and React") assert 'python' in entities['technologies'] @@ -68,28 +63,28 @@ class TestTagger: """Test auto-tagging.""" def test_keyword_tagging(self): - from extractors.tagger import auto_tag + from src.extractors.tagger import auto_tag tags = auto_tag("Working on a Python bug fix") assert 'python' in tags assert 'bug' in tags def test_pattern_tagging(self): - from extractors.tagger import auto_tag + from src.extractors.tagger import auto_tag tags = auto_tag("How to fix this error?") assert 'error' in tags assert 'question' in tags def test_user_tags(self): - from extractors.tagger import auto_tag + from src.extractors.tagger import auto_tag tags = auto_tag("Some note", user_tags=['important', 'review']) assert 'important' in tags assert 'review' in tags def test_deny_list(self): - from extractors.tagger import Tagger + from src.extractors.tagger import Tagger tagger = Tagger() tags = tagger.tag("test", user_tags=['password', 'valid_tag']) @@ -110,7 +105,7 @@ def _ollama_config(): @patch('requests.post') def test_embed_creation(self, mock_post): - from embedder import OllamaEmbedder + from src.embedder import OllamaEmbedder mock_response = Mock() mock_response.json.return_value = {'embedding': [0.1] * 768} @@ -125,7 +120,7 @@ def test_embed_creation(self, mock_post): @patch('requests.post') def test_batch_embedding(self, mock_post): - from embedder import OllamaEmbedder + from src.embedder import OllamaEmbedder mock_response = Mock() mock_response.json.return_value = {'embedding': [0.1] * 768} @@ -143,7 +138,7 @@ class TestAnalytics: """Test analytics functions.""" def test_trend_analyzer_init(self): - from analytics.trends import TrendAnalyzer + from src.analytics.trends import TrendAnalyzer analyzer = TrendAnalyzer(weeks=4) assert analyzer.weeks == 4 @@ -153,13 +148,13 @@ class TestMemoryFormatting: """Test memory formatting functions.""" def test_format_empty_list(self): - from main import format_memory_list + from src.main import format_memory_list result = format_memory_list([]) assert "No memories found" in result def test_format_memory_list(self): - from main import format_memory_list + from src.main import format_memory_list memories = [ { @@ -167,7 +162,7 @@ def test_format_memory_list(self): 'source': 'test', 'content': 'Test content', 'tags': ['test'], - 'created_at': datetime.now() + 'created_at': datetime.now(timezone.utc) } ] @@ -180,9 +175,9 @@ def test_format_memory_list(self): class TestDatabaseQueries: """Test database query functions with realistic cursor mocks.""" - @patch('db.queries.get_db_cursor') + @patch('src.db.queries.get_db_cursor') def test_search_memories(self, mock_cursor): - from db import queries + from src.db import queries mock_ctx = Mock() mock_ctx.__enter__ = Mock(return_value=mock_ctx) @@ -193,9 +188,9 @@ def test_search_memories(self, mock_cursor): results = queries.search_memories("test", limit=5) assert isinstance(results, list) - @patch('db.queries.get_db_cursor') + @patch('src.db.queries.get_db_cursor') def test_get_memory_stats(self, mock_cursor): - from db import queries + from src.db import queries mock_ctx = Mock() mock_ctx.__enter__ = Mock(return_value=mock_ctx) diff --git a/tests/test_extended_hermes_importers.py b/tests/test_extended_hermes_importers.py index f37d62e..10fbd2f 100644 --- a/tests/test_extended_hermes_importers.py +++ b/tests/test_extended_hermes_importers.py @@ -4,9 +4,9 @@ import pytest -from importers.hermes_context import discover_allowlisted_context -from importers.hermes_operations import HermesCronImporter, HermesSkillImporter -from importers.hermes_sessions import HermesSessionImporter +from src.importers.hermes_context import discover_allowlisted_context +from src.importers.hermes_operations import HermesCronImporter, HermesSkillImporter +from src.importers.hermes_sessions import HermesSessionImporter def test_context_import_requires_allowlist_and_blocks_escape(tmp_path): diff --git a/tests/test_hermes_markdown_importer.py b/tests/test_hermes_markdown_importer.py index 2496387..e817cb0 100644 --- a/tests/test_hermes_markdown_importer.py +++ b/tests/test_hermes_markdown_importer.py @@ -1,7 +1,7 @@ """Tests for safe bootstrap import of Hermes markdown memory.""" -from importers.base import ImportSource, hash_content -from importers.hermes_markdown import HermesMarkdownImporter, discover_default_hermes_memory +from src.importers.base import ImportSource, hash_content +from src.importers.hermes_markdown import HermesMarkdownImporter, discover_default_hermes_memory def test_importer_preserves_sections_authority_and_hashes(tmp_path): diff --git a/tests/test_hermes_plugin_install.py b/tests/test_hermes_plugin_install.py index 2dedaa1..3d97f5d 100644 --- a/tests/test_hermes_plugin_install.py +++ b/tests/test_hermes_plugin_install.py @@ -1,4 +1,3 @@ -from pathlib import Path from src.cli import install_hermes_cmd diff --git a/tests/test_hermes_provider_contract.py b/tests/test_hermes_provider_contract.py index 46c531f..0c1b332 100644 --- a/tests/test_hermes_provider_contract.py +++ b/tests/test_hermes_provider_contract.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import json import sys import types from pathlib import Path @@ -31,6 +32,44 @@ def test_digest_is_deterministic(): assert module._digest("a", "b") != module._digest("b", "a") +def test_client_loads_generated_key_and_authenticates(tmp_path, monkeypatch): + module = _load_provider_module() + config_dir = tmp_path / ".config" / "openbrain" + config_dir.mkdir(parents=True) + (config_dir / ".env").write_text( + "OPENBRAIN_URL=http://127.0.0.1:8000\n" + "OPENBRAIN_API_KEY=generated-secret\n", + encoding="utf-8", + ) + monkeypatch.setenv("OPENBRAIN_CONFIG_DIR", str(config_dir)) + monkeypatch.delenv("OPENBRAIN_API_KEY", raising=False) + + captured = {} + + class Response: + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def read(self): + return json.dumps({"ok": True}).encode() + + def fake_urlopen(request, timeout): + captured["authorization"] = request.get_header("Authorization") + return Response() + + monkeypatch.setattr(module.urllib.request, "urlopen", fake_urlopen) + settings = module._load_openbrain_environment() + module._Client(settings["OPENBRAIN_URL"], api_key=settings["OPENBRAIN_API_KEY"]).request( + "GET", + "/health", + ) + + assert captured["authorization"] == "Bearer generated-secret" + + def test_open_session_preserves_internal_session_id(): module = _load_provider_module() provider = module.OpenBrainMemoryProvider() diff --git a/tests/test_host_adapters.py b/tests/test_host_adapters.py index 72397dd..1acaed9 100644 --- a/tests/test_host_adapters.py +++ b/tests/test_host_adapters.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import httpx @@ -36,7 +37,32 @@ def test_install_and_uninstall_env_file(tmp_path: Path): assert installed == path assert "OPENBRAIN_API_KEY=secret" in path.read_text() - assert oct(path.stat().st_mode & 0o777) == "0o600" + if os.name == "nt": + import win32api + import win32con + import win32security + + token = win32security.OpenProcessToken( + win32api.GetCurrentProcess(), + win32con.TOKEN_QUERY, + ) + current_user_sid = win32security.ConvertSidToStringSid( + win32security.GetTokenInformation(token, win32security.TokenUser)[0] + ) + descriptor = win32security.GetFileSecurity( + str(path), + win32security.DACL_SECURITY_INFORMATION, + ) + dacl = descriptor.GetSecurityDescriptorDacl() + trustee_sids = { + win32security.ConvertSidToStringSid(dacl.GetAce(index)[2]) + for index in range(dacl.GetAceCount()) + } + assert current_user_sid in trustee_sids + assert "S-1-1-0" not in trustee_sids + assert "S-1-5-11" not in trustee_sids + else: + assert oct(path.stat().st_mode & 0o777) == "0o600" assert uninstall_env_file(path) is True assert uninstall_env_file(path) is False diff --git a/tests/test_import_rollback.py b/tests/test_import_rollback.py index ed724c2..1b93732 100644 --- a/tests/test_import_rollback.py +++ b/tests/test_import_rollback.py @@ -1,14 +1,15 @@ from uuid import uuid4 import pytest +from pydantic import ValidationError -from api.imports import ImportRollbackRequest, ImportRollbackResponse +from src.api.imports import ImportRollbackRequest, ImportRollbackResponse def test_rollback_request_requires_actor_and_reason(): - with pytest.raises(Exception): + with pytest.raises(ValidationError): ImportRollbackRequest(actor="", reason="cleanup") - with pytest.raises(Exception): + with pytest.raises(ValidationError): ImportRollbackRequest(actor="operator", reason="") diff --git a/tests/test_import_runner.py b/tests/test_import_runner.py index 828f96d..5c01d97 100644 --- a/tests/test_import_runner.py +++ b/tests/test_import_runner.py @@ -4,8 +4,8 @@ import pytest -from importers.base import ImportAdapter, ImportCandidate, ImportSource, hash_content -from importers.runner import run_import +from src.importers.base import ImportAdapter, ImportCandidate, ImportSource, hash_content +from src.importers.runner import run_import class StaticAdapter(ImportAdapter): @@ -36,16 +36,16 @@ def test_import_runner_dry_run_records_preview(monkeypatch): updates = [] monkeypatch.setattr( - "importers.runner.create_import_run", + "src.importers.runner.create_import_run", lambda *args, **kwargs: {"id": run_id}, ) - monkeypatch.setattr("importers.runner.seen_external_hashes", lambda _: set()) + monkeypatch.setattr("src.importers.runner.seen_external_hashes", lambda _: set()) monkeypatch.setattr( - "importers.runner.record_import_candidate", + "src.importers.runner.record_import_candidate", lambda run, candidate, **kwargs: recorded.append((candidate, kwargs)) or True, ) monkeypatch.setattr( - "importers.runner.update_import_run", + "src.importers.runner.update_import_run", lambda run, **kwargs: updates.append(kwargs) or {"id": run}, ) @@ -67,13 +67,13 @@ def test_import_runner_non_dry_run_requires_sealing(monkeypatch): run_id = uuid4() updates = [] monkeypatch.setattr( - "importers.runner.create_import_run", + "src.importers.runner.create_import_run", lambda *args, **kwargs: {"id": run_id}, ) - monkeypatch.setattr("importers.runner.seen_external_hashes", lambda _: set()) - monkeypatch.setattr("importers.runner.record_import_candidate", lambda *args, **kwargs: True) + monkeypatch.setattr("src.importers.runner.seen_external_hashes", lambda _: set()) + monkeypatch.setattr("src.importers.runner.record_import_candidate", lambda *args, **kwargs: True) monkeypatch.setattr( - "importers.runner.update_import_run", + "src.importers.runner.update_import_run", lambda run, **kwargs: updates.append(kwargs) or {"id": run}, ) @@ -86,7 +86,7 @@ def test_import_runner_non_dry_run_requires_sealing(monkeypatch): def test_import_runner_rejects_resume_after_source_change(monkeypatch): run_id = uuid4() monkeypatch.setattr( - "importers.runner.get_import_run", + "src.importers.runner.get_import_run", lambda _: { "id": run_id, "status": "running", @@ -105,7 +105,7 @@ def test_import_runner_rejects_resume_after_source_change(monkeypatch): def test_import_runner_rejects_sealed_resume(monkeypatch): run_id = uuid4() monkeypatch.setattr( - "importers.runner.get_import_run", + "src.importers.runner.get_import_run", lambda _: {"id": run_id, "status": "sealed", "config": {"dry_run": False}}, ) @@ -123,19 +123,19 @@ def test_import_runner_skips_already_recorded_candidate(monkeypatch): candidate = list(adapter.discover())[0] monkeypatch.setattr( - "importers.runner.create_import_run", + "src.importers.runner.create_import_run", lambda *args, **kwargs: {"id": run_id}, ) monkeypatch.setattr( - "importers.runner.seen_external_hashes", + "src.importers.runner.seen_external_hashes", lambda _: {(candidate.external_id, candidate.external_hash)}, ) monkeypatch.setattr( - "importers.runner.record_import_candidate", + "src.importers.runner.record_import_candidate", lambda *args, **kwargs: pytest.fail("duplicate candidate should not be recorded"), ) monkeypatch.setattr( - "importers.runner.update_import_run", + "src.importers.runner.update_import_run", lambda run, **kwargs: {"id": run}, ) @@ -144,4 +144,4 @@ def test_import_runner_skips_already_recorded_candidate(monkeypatch): assert result.status == "staged" assert result.records_seen == 1 assert result.records_imported == 0 - assert result.records_merged == 1 \ No newline at end of file + assert result.records_merged == 1 diff --git a/tests/test_installation_productization.py b/tests/test_installation_productization.py index 60068f1..fee344d 100644 --- a/tests/test_installation_productization.py +++ b/tests/test_installation_productization.py @@ -35,8 +35,11 @@ def fail(*args, **kwargs): def test_installer_wires_hermes_and_runs_doctor(): script = Path("install.sh").read_text(encoding="utf-8") assert "OPENBRAIN_INSTALL_HERMES" in script - assert "openbrain install-hermes --force" in script - assert "openbrain doctor" in script + assert script.index('"$OPENBRAIN_BIN" configure') < script.index( + '"$OPENBRAIN_BIN" install-hermes --force' + ) + assert '"$OPENBRAIN_BIN" install-hermes --force' in script + assert '"$OPENBRAIN_BIN" doctor' in script def test_installer_preserves_isolation_and_supports_forks(): diff --git a/tests/test_proposal_api_contracts.py b/tests/test_proposal_api_contracts.py index 471876e..4fd6c67 100644 --- a/tests/test_proposal_api_contracts.py +++ b/tests/test_proposal_api_contracts.py @@ -3,7 +3,7 @@ import pytest from pydantic import ValidationError -from api.proposals import ProposalActorRequest, ProposalReviewRequest +from src.api.proposals import ProposalActorRequest, ProposalReviewRequest def test_acceptance_allows_optional_note() -> None: diff --git a/tests/test_release_readiness.py b/tests/test_release_readiness.py index 3b4a6dd..b3edc8c 100644 --- a/tests/test_release_readiness.py +++ b/tests/test_release_readiness.py @@ -1,6 +1,6 @@ from __future__ import annotations -from release.readiness import evaluate_release_readiness +from src.release.readiness import evaluate_release_readiness def production_env(**overrides: str) -> dict[str, str]: From ec691f95772f96ce65b351ef01b7d127832012a8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Jul 2026 09:26:31 +0200 Subject: [PATCH 2/3] fix: support module-based CLI execution --- src/cli/__main__.py | 7 +++++++ tests/test_code_quality_regressions.py | 11 +++++++++++ 2 files changed, 18 insertions(+) create mode 100644 src/cli/__main__.py diff --git a/src/cli/__main__.py b/src/cli/__main__.py new file mode 100644 index 0000000..d1f49b9 --- /dev/null +++ b/src/cli/__main__.py @@ -0,0 +1,7 @@ +"""Run the Open Brain CLI with ``python -m src.cli``.""" + +from src.cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_code_quality_regressions.py b/tests/test_code_quality_regressions.py index 808fc0d..f277b6f 100644 --- a/tests/test_code_quality_regressions.py +++ b/tests/test_code_quality_regressions.py @@ -221,6 +221,17 @@ def test_runtime_and_hermes_plugin_versions_match_distribution() -> None: assert plugin["version"] == project_version() +def test_package_cli_module_is_executable() -> None: + result = subprocess.run( + [sys.executable, "-m", "src.cli", "--version"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + assert result.stdout.strip() == f"Open Brain {project_version()}" + + def test_generated_distribution_metadata_is_not_tracked_as_source() -> None: tracked = subprocess.run( ["git", "ls-files", "openbrain.egg-info"], From cc4261622be00b913d986165431be64ec4b742f1 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Jul 2026 09:29:35 +0200 Subject: [PATCH 3/3] test: isolate Hermes credential fixture --- tests/test_hermes_provider_contract.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_hermes_provider_contract.py b/tests/test_hermes_provider_contract.py index 0c1b332..c56b485 100644 --- a/tests/test_hermes_provider_contract.py +++ b/tests/test_hermes_provider_contract.py @@ -43,6 +43,7 @@ def test_client_loads_generated_key_and_authenticates(tmp_path, monkeypatch): ) monkeypatch.setenv("OPENBRAIN_CONFIG_DIR", str(config_dir)) monkeypatch.delenv("OPENBRAIN_API_KEY", raising=False) + monkeypatch.chdir(tmp_path) captured = {}