diff --git a/.env.example b/.env.example
index f65de21..6384489 100644
--- a/.env.example
+++ b/.env.example
@@ -2,18 +2,26 @@
# Give your OpenClaw AI agent a physical robot body!
# ==============================================================================
-# REQUIRED: OpenAI API Key
+# OPTIONAL: OpenAI API Key (legacy cloud vision fallback only)
# ==============================================================================
# Get your key at: https://platform.openai.com/api-keys
-# Requires Realtime API access
OPENAI_API_KEY=sk-your-openai-key
+# ==============================================================================
+# REQUIRED: Always-hot local speech-to-text
+# ==============================================================================
+# OpenAI-compatible /v1/audio/transcriptions service on xeon-serv.
+STT_BASE_URL=http://speech-host.local:8890/v1
+STT_MODEL=distil-large-v3
+STT_LANGUAGE=en
+# STT_API_KEY=optional-local-bearer-token
+
# ==============================================================================
# REQUIRED: OpenClaw Gateway
# ==============================================================================
# The URL where your OpenClaw gateway is running
# If running on the same machine as the robot, use the host machine's IP
-OPENCLAW_GATEWAY_URL=http://192.168.1.100:18789
+OPENCLAW_GATEWAY_URL=http://openclaw-host.local:18789
# Your OpenClaw gateway authentication token
# Find this in ~/.openclaw/openclaw.json under gateway.token
@@ -23,18 +31,52 @@ OPENCLAW_TOKEN=your-gateway-token
OPENCLAW_AGENT_ID=main
# Session key for conversation context - IMPORTANT!
-# Use "main" (default) to share context with WhatsApp and other DM channels
-# This allows the robot to be aware of all your conversations
-OPENCLAW_SESSION_KEY=main
+# Keep the robot in a dedicated session while using the same OpenClaw agent.
+OPENCLAW_SESSION_KEY=reachy
+
+# Reachy daemon. These settings let ClawBody run on xeon-serv while using
+# Reachy's remote WebRTC media and motion connection.
+ROBOT_HOST=reachy-mini.local
+ROBOT_PORT=8000
+ROBOT_CONNECTION_MODE=network
# ==============================================================================
-# OPTIONAL: Voice Settings
+# REQUIRED: Voice Settings
# ==============================================================================
-# OpenAI Realtime voice (alloy, echo, fable, onyx, nova, shimmer, cedar)
-OPENAI_VOICE=cedar
+# Chatterbox is the default local provider. Its service uses a named voice
+# profile installed on xeon-serv rather than receiving a secret/reference file
+# on every request.
+TTS_PROVIDER=chatterbox
+CHATTERBOX_URL=http://speech-host.local:8890/v1/audio/speech
+CHATTERBOX_VOICE=default
+# Software playback gain before Reachy's hardware/app volume. Start at 1.0;
+# values above 1.0 boost quiet voices with clipping protection.
+AUDIO_OUTPUT_GAIN=1.0
-# OpenAI model for Realtime API
-OPENAI_MODEL=gpt-4o-realtime-preview-2024-12-17
+# If both values are present, ElevenLabs is the automatic Chatterbox fallback.
+# Set TTS_PROVIDER=elevenlabs to make it primary.
+# ELEVENLABS_API_KEY=...
+# ELEVENLABS_VOICE_ID=...
+# ELEVENLABS_MODEL_ID=eleven_flash_v2_5
+
+# Local VAD tuning
+VAD_RMS_THRESHOLD=0.022
+VAD_ACTIVATION_MS=160
+VAD_PREFIX_MS=250
+VAD_SILENCE_MS=650
+VAD_MIN_SPEECH_MS=300
+VAD_MAX_SPEECH_SECONDS=20
+VAD_REQUIRE_HARDWARE_SPEECH=true
+
+# Optional wake phrase gate. When enabled, only transcripts beginning with the
+# configured phrase are sent to OpenClaw; the phrase is stripped from commands.
+ENABLE_WAKE_PHRASE=false
+WAKE_PHRASE=Hey Asmo
+
+# Quiet mail-attention indicator and head double-tap playback.
+ENABLE_ATTENTION_QUEUE=true
+ATTENTION_API_URL=http://192.168.1.238:18790
+ATTENTION_POLL_SECONDS=60
# ==============================================================================
# OPTIONAL: Features
@@ -42,3 +84,11 @@ OPENAI_MODEL=gpt-4o-realtime-preview-2024-12-17
# Enable/disable features (true/false)
ENABLE_CAMERA=true
ENABLE_OPENCLAW_TOOLS=true
+ENABLE_FACE_TRACKING=true
+HEAD_TRACKER_TYPE=daemon
+FACE_TRACKING_WEIGHT=0.7
+ENABLE_SOUND_TRACKING=true
+
+# Dedicated body-session latency. "off" is the instant conversational mode;
+# it does not change reasoning settings for Discord, cron, or other sessions.
+OPENCLAW_THINKING=off
diff --git a/.gitignore b/.gitignore
index 49ad75e..ee28adb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
# Environment and secrets
.env
+.private/
*.env.local
# Python
diff --git a/EMOTIONS.md b/EMOTIONS.md
new file mode 100644
index 0000000..c4d9603
--- /dev/null
+++ b/EMOTIONS.md
@@ -0,0 +1,87 @@
+# Reachy emotion map
+
+ClawBody exposes the 81 expressive recordings from Pollen's Reachy Mini
+emotions dataset. Exact numbered names always select that recording; a family
+name or conversational alias selects the first (restrained default) variant.
+
+The four additional dataset recordings are utilities, not emotions:
+`mini-deep-sleep`, `toc-toc-toc`, `waiting`, and `wake-mini-up`.
+
+## Positive and social
+
+- amazed: `amazed1` (astonished, impressed, wow)
+- attentive: `attentive1`, `attentive2` (alert, focused, listening)
+- calming: `calming1` (soothing, reassuring)
+- cheerful: `cheerful1` (happy, upbeat)
+- come: `come1` (come here, beckon, invite)
+- curious: `curious1`
+- enthusiastic: `enthusiastic1`, `enthusiastic2` (excited, eager)
+- grateful: `grateful1` (thankful)
+- helpful: `helpful1`, `helpful2` (assist)
+- laughing: `laughing1`, `laughing2` (laugh, amused)
+- loving: `loving1` (love, affectionate)
+- proud: `proud1`, `proud2`, `proud3`
+- relief: `relief1`, `relief2` (relieved)
+- serenity: `serenity1` (peaceful)
+- success: `success1`, `success2` (victory, celebrating)
+- understanding: `understanding1`, `understanding2` (got it, comprehend)
+- welcoming: `welcoming1`, `welcoming2` (welcome, greet)
+- yes: `yes1`, `yes_sad1` (agree, affirmative; “sad yes” selects `yes_sad1`)
+
+## Thinking and uncertainty
+
+- anxiety: `anxiety1` (anxious, nervous, worried)
+- confused: `confused1` (puzzled)
+- incomprehensible: `incomprehensible2` (baffled, don't understand)
+- inquiring: `inquiring1`, `inquiring2`, `inquiring3` (questioning)
+- lost: `lost1`
+- oops: `oops1`, `oops2` (mistake)
+- surprised: `surprised1`, `surprised2` (startled)
+- thoughtful: `thoughtful1`, `thoughtful2` (thinking, contemplative)
+- uncertain: `uncertain1` (unsure, doubtful)
+
+## Low energy and vulnerable
+
+- boredom: `boredom1`, `boredom2` (bored)
+- downcast: `downcast1` (dejected)
+- exhausted: `exhausted1` (wiped out)
+- lonely: `lonely1`
+- resigned: `resigned1` (give up, accept defeat)
+- sad: `sad1`, `sad2` (unhappy)
+- shy: `shy1` (bashful)
+- sleep: `sleep1` (sleepy)
+- tired: `tired1`
+- uncomfortable: `uncomfortable1` (awkward, uneasy)
+
+## Negative and defensive
+
+- contempt: `contempt1` (disdain, dismissive)
+- disgusted: `disgusted1` (grossed out)
+- displeased: `displeased1`, `displeased2` (dissatisfied)
+- fear: `fear1` (afraid)
+- frustrated: `frustrated1`
+- furious: `furious1` (livid)
+- go away: `go_away1` (shoo, leave)
+- impatient: `impatient1`, `impatient2`
+- indifferent: `indifferent1` (meh, apathetic)
+- irritated: `irritated1`, `irritated2` (annoyed)
+- no: `no1`, `no_excited1`, `no_sad1` (decline, refuse; “excited no” and “sad no” select their exact variants)
+- rage: `rage1` (enraged)
+- reprimand: `reprimand1`, `reprimand2`, `reprimand3` (scold, admonish)
+- scared: `scared1` (frightened)
+
+## Theatrical and movement reactions
+
+- dance: `dance1`, `dance2`, `dance3` (legacy emotion-dataset motions; explicit dance requests use the separate official dance library)
+- dying: `dying1` (dramatic death)
+- electric: `electric1` (energized)
+
+## Selection rules
+
+- Exact names win: “express proud three” selects `proud3`.
+- Family names choose the first installed variant: “act proud” selects `proud1`.
+- Common aliases resolve to a family: “show victory” selects `success1`.
+- Authored emotion moves get exclusive body control; face tracking, sound tracking,
+ and speech wobble pause until the move completes.
+- New upstream recordings appear as unmapped until intentionally assigned to a
+ family, preventing silent or nonsensical gesture selection.
diff --git a/NATIVE_APP_REVIEW.md b/NATIVE_APP_REVIEW.md
new file mode 100644
index 0000000..994582e
--- /dev/null
+++ b/NATIVE_APP_REVIEW.md
@@ -0,0 +1,67 @@
+# Native Reachy App Contract Review
+
+Reviewed against the installed Reachy Mini 1.9 SDK and its bundled
+`reachy-mini-app-assistant` validator.
+
+## Confirmed contract
+
+- The project name is `clawbody`; the validator derives the package from the
+ entry point and therefore expects class `ReachyMiniOpenclaw` in
+ `reachy_mini_openclaw.main`.
+- The native class must explicitly inherit `reachy_mini.ReachyMiniApp`.
+- The entry point must be:
+
+ ```toml
+ [project.entry-points."reachy_mini_apps"]
+ clawbody = "reachy_mini_openclaw.main:ReachyMiniOpenclaw"
+ ```
+
+- The SDK owns the `ReachyMini` context, creates a `threading.Event`, and calls
+ `run(reachy_mini, stop_event)` through `ReachyMiniApp.wrapped_run()`.
+- The app must stop when that event is set and must not close/disconnect the
+ SDK-owned robot itself.
+- The existing README frontmatter, root `index.html`, root `style.css`, and src
+ package layout satisfy the validator's static metadata/layout requirements.
+- The official validator finishes by installing the project into an isolated
+ venv, verifying the entry point, and uninstalling it.
+
+## Current defects to correct
+
+1. `ClawBodyApp` neither has the validator-required name nor inherits
+ `ReachyMiniApp`. The current validator fails before installation with the
+ expected entry point `reachy_mini_openclaw.main:ReachyMiniOpenclaw`.
+2. The native wrapper catches and suppresses every exception. That prevents the
+ app manager from observing a failed app and undermines watchdog/restart
+ behavior. Cleanup should occur in `finally`, then the failure must propagate.
+3. The external stop event is only polled by the record/play loops. The
+ conversation handler remains in the `asyncio.gather()` set, so a dashboard
+ stop can leave `run()` waiting indefinitely. A stop watcher must cancel all
+ app tasks promptly.
+4. `ClawBodyCore.stop()` is synchronous but tries to run the asynchronous
+ gateway disconnect with `run_until_complete()` on the current event loop.
+ When called from the native wrapper after `run_until_complete()`, this is
+ fragile; when called while a loop is active, it is invalid. Async shutdown
+ needs a single owning loop and an idempotent lifecycle.
+5. Configuration validation calls `sys.exit(1)` inside an embeddable app class.
+ Native lifecycle errors should raise typed exceptions so the SDK can record
+ `app.error` and manage failure correctly.
+6. Native camera tracking currently builds its daemon URL from
+ `ROBOT_HOST`/`ROBOT_PORT`. On the Wireless body, the SDK-owned robot is local
+ and the daemon-side HTTP API should use localhost (or an explicit native
+ daemon URL), not the remote workstation default.
+7. The package deliberately depends on remote xeon STT/TTS and the existing
+ OpenClaw gateway. Startup must tolerate those services being temporarily
+ unavailable and reconnect with bounded backoff rather than exiting or
+ pretending the app is healthy.
+
+## Required implementation shape
+
+- Introduce `ReachyMiniOpenclaw(ReachyMiniApp)` with the exact entry point.
+- Keep `ClawBodyCore` reusable for CLI/simulator operation.
+- Give the core an idempotent async shutdown path and an explicit external-stop
+ watcher.
+- Let fatal native exceptions propagate after cleanup.
+- Add contract tests for inheritance, entry-point loading, stop behavior,
+ cleanup idempotency, and reconnect/backoff behavior.
+- Preserve the existing `main` agent and `agent:main:reachy` session. Native
+ packaging changes transport/lifecycle only; it must not create Asmo(deux).
diff --git a/NATIVE_DEPLOY_TODO.md b/NATIVE_DEPLOY_TODO.md
new file mode 100644
index 0000000..59b29cc
--- /dev/null
+++ b/NATIVE_DEPLOY_TODO.md
@@ -0,0 +1,80 @@
+# Reachy Native Deployment TODO
+
+Last checked: 2026-07-22
+
+Goal: package and deploy ClawBody as a native Reachy Mini app that starts and recovers without the laptop, while continuing to use the existing OpenClaw `main` agent and `agent:main:reachy` session. Do not create a second Asmo agent.
+
+## Work queue
+
+- [x] **Consolidate reviewer findings and inspect the current native app contract**
+ - Capture the actionable Claude review findings in the repo.
+ - Verify the expected `ReachyMiniApp` class, lifecycle hooks, metadata, and runtime assumptions against the installed Reachy 1.9 tooling.
+ - Exit condition: findings are recorded and the exact contract changes are identified.
+ - Evidence: `NATIVE_APP_REVIEW.md`; installed Reachy 1.9 validator reproduces the entry-point/class failure and documents the required lifecycle corrections.
+
+- [x] **Correct the native Reachy app entry point and lifecycle**
+ - Fix the app class/entry-point contract.
+ - Ensure startup, stop, cleanup, and robot-resource ownership are safe and idempotent.
+ - Preserve the existing xeon STT/TTS and `agent:main:reachy` architecture.
+ - Exit condition: native entry point imports and initializes under the official tooling.
+ - Evidence: `ReachyMiniOpenclaw(ReachyMiniApp)` is the package entry point; external stop events cancel all app tasks, shutdown is async/idempotent, native failures propagate, and native camera tracking uses localhost.
+
+- [x] **Add reconnect and watchdog resilience**
+ - Recover from Reachy daemon, WebRTC, xeon voice service, and OpenClaw gateway interruptions without a laptop.
+ - Use bounded backoff and clean shutdown; do not create restart storms.
+ - Exit condition: automated tests cover disconnect/reconnect and process restart behavior.
+ - Evidence: gateway reconnect, transient STT/TTS retry, microphone/speaker recovery backoff, and native failure propagation to the Reachy supervisor are covered by the 33-test suite.
+
+- [x] **Make native configuration and secrets deployment-safe**
+ - Remove LAN-specific or workstation-only assumptions from the package.
+ - Keep secrets out of git and Hugging Face history.
+ - Document the minimum runtime configuration for Reachy, xeon STT/TTS, and the existing OpenClaw gateway/session.
+ - Exit condition: a clean install can be configured without editing source.
+ - Evidence: service endpoints are explicit required configuration, deployment environment overrides local `.env`, native configuration is documented in `README.md`, and secrets remain environment-only.
+
+- [x] **Run local validation and final code review**
+ - Run the complete test suite.
+ - Run `reachy-mini-app-assistant check` (or the current official equivalent).
+ - Review the final diff for safety, secret leakage, lifecycle correctness, and accidental creation of an Asmo(deux).
+ - Exit condition: validation passes with no unresolved critical findings.
+ - Evidence: UV-managed CPython 3.13.12; 35 tests pass; the Reachy 1.9 app assistant passes its clean install, entry-point, and uninstall checks; package builds succeed; artifact and workspace scans found no credential material; final review fixed executor-thread leakage, un-awaited CLI/Gradio shutdown, and blocking async startup sleeps.
+
+- [x] **Package and publish a private Hugging Face Space**
+ - Use the Hugging Face credentials in 1Password vault `Asmo`, item `3nm3rm35entwmzoc64bznvdbma` at runtime only.
+ - Create/update the private native Reachy app Space and push the validated package.
+ - Never print or persist the token in logs, repo files, shell history, or memory.
+ - Exit condition: the private Space exists and exposes the expected app revision.
+ - Evidence: the scoped write token was read from 1Password at runtime only; the official Reachy 1.9 publish check passed; private Space `asmolebot/clawbodylocal` was created and verified at revision `dc3d1ae0c2d4386ed186476edc6efb388d08c97d` with the expected native app files and no secret-like filenames. The original `asmolebot/clawbody` identifier resolves to the renamed Space through the authenticated Hub API.
+
+- [ ] **Install and configure the app on Reachy Mini**
+ - HOLD: do not begin this item until Chris explicitly signs off on the final deployment.
+ - Install through the Reachy app API/tooling.
+ - Configure it as the startup app if supported by the installed Reachy version.
+ - Keep deployment reversible and record the prior app/startup state.
+ - Exit condition: the app launches on Reachy without the laptop console.
+
+- [ ] **Run laptop-independent end-to-end acceptance tests**
+ - Verify microphone -> xeon STT -> existing OpenClaw Asmo session -> xeon TTS -> speaker.
+ - Verify camera/face/sound tracking, representative emotions, and at least one safe dance.
+ - Restart the app/daemon and confirm recovery.
+ - Exit condition: a complete spoken turn and motion test pass while the laptop is closed/disconnected.
+
+- [ ] **Commit, push, and update the upstream draft PR**
+ - Commit only reviewed source/docs/config changes; exclude private deployment state and credentials.
+ - Push the existing branch and update the draft PR with native-deployment validation results.
+ - Exit condition: clean worktree, pushed commit, and PR reflects the deployed revision.
+
+- [ ] **Begin the separate body-latency/bootstrap optimization**
+ - Only after native deployment is stable, slim the Reachy session bootstrap/tool surface and add first-token/first-audio timing.
+ - Do not create a second agent personality or duplicate long-term memory.
+ - Exit condition: a measured optimization plan is recorded from the native deployment baseline.
+
+## Heartbeat rules
+
+- Advance at most one unchecked item per heartbeat.
+- Read current repo/runtime state before acting; do not redo completed work.
+- Mark an item complete only when its exit condition is evidenced.
+- Continue safe local implementation and validation autonomously.
+- Publishing/staging the private Space is authorized. Installing, launching, or changing startup state on Reachy requires Chris's explicit final sign-off.
+- Report only completion, a new blocker, deployment/recovery state, or a request that genuinely needs Chris.
+- If no material state changed, stay quiet.
diff --git a/README.md b/README.md
index 9a8d5a3..c0c714c 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,11 @@
---
-title: ClawBody
+title: ClawBody Local
emoji: 🦞
colorFrom: red
colorTo: purple
sdk: static
pinned: false
-short_description: OpenClaw AI with robot body and face tracking
+short_description: Asmo's local OpenClaw body for Reachy Mini
tags:
- reachy_mini
- reachy_mini_python_app
@@ -15,7 +15,7 @@ tags:
- ai-assistant
- voice-assistant
- robotics
- - openai-realtime
+ - local-speech
- conversational-ai
- physical-ai
- robot-body
@@ -31,11 +31,14 @@ tags:
- human-robot-interaction
---
-# 🦞🤖 ClawBody
+# 🦞🤖 ClawBody Local
+
+See [EMOTIONS.md](EMOTIONS.md) for the complete 81-move Reachy emotion map,
+semantic families, aliases, and selection behavior.
**Give your OpenClaw AI agent a physical robot body!**
-ClawBody combines OpenClaw's AI intelligence with Reachy Mini's expressive robot body, using OpenAI's Realtime API for ultra-responsive voice conversation. Your OpenClaw assistant (Clawson) can now see, hear, speak, and move in the physical world.
+ClawBody Local is Asmo-owned infrastructure that combines OpenClaw's AI intelligence with Reachy Mini's expressive robot body. Local STT transcribes the microphone, OpenClaw produces the actual response, and Chatterbox or ElevenLabs gives it a voice.

@@ -48,21 +51,18 @@ ClawBody combines OpenClaw's AI intelligence with Reachy Mini's expressive robot
**The robot looks at you when you speak!**
-ClawBody now includes real-time face tracking that makes conversations feel natural and engaging:
+ClawBody uses Reachy Mini 1.9's daemon-side face tracking and microphone-array
+direction-of-arrival support, avoiding a duplicate detector on the OpenClaw host:
-- **Automatic Face Detection**: Uses MediaPipe or YOLO to detect faces at 25Hz
+- **Automatic Face Detection**: Runs in Reachy's daemon and blends with expressive movement
- **Smooth Head Tracking**: Robot smoothly follows your face as you move
-- **Natural Eye Contact**: Maintains engagement during conversation
-- **Graceful Fallback**: Smoothly returns to neutral position when you leave
+- **Sound Tracking**: Turns conservatively toward speech when no face is visible
+- **Visual Questions**: Attaches a current camera frame to OpenClaw when asked what it sees
```bash
# Face tracking is enabled by default
clawbody
-# Choose your tracker (MediaPipe is lighter, YOLO is more accurate)
-clawbody --head-tracker mediapipe
-clawbody --head-tracker yolo
-
# Disable if needed
clawbody --no-face-tracking
```
@@ -93,7 +93,7 @@ clawbody --gradio
## ✨ Features
- **👁️ Face Tracking**: Robot tracks your face and maintains eye contact during conversation
-- **🎤 Real-time Voice Conversation**: OpenAI Realtime API for sub-second response latency
+- **🎤 Local Voice Pipeline**: Always-hot STT and pluggable Chatterbox/ElevenLabs speech
- **🧠 OpenClaw Intelligence**: Your responses come from OpenClaw with full tool access
- **👀 Vision**: See through the robot's camera and describe the environment
- **💃 Expressive Movements**: Natural head movements, emotions, dances, and audio-driven wobble
@@ -120,9 +120,9 @@ clawbody --gradio
┌─────────────────────────────────────────────────┼───────────────┐
│ ClawBody │ │
│ ┌─────────────────────────────────────────────┼────────────┐ │
-│ │ OpenAI Realtime API Handler │ │ │
-│ │ • Speech recognition (Whisper) │ │ │
-│ │ • Text-to-speech (voices) ─┘ │ │
+│ │ Provider-based Voice Pipeline │ │ │
+│ │ • Local STT + local VAD │ │ │
+│ │ • Chatterbox / ElevenLabs TTS ─┘ │ │
│ │ • Audio analysis → head wobble │ │
│ └─────────────────────────────────────────────────────────┘ │
│ │ │
@@ -156,7 +156,8 @@ clawbody --gradio
- Python 3.11+
- [Reachy Mini SDK](https://github.com/pollen-robotics/reachy_mini) installed
- [OpenClaw](https://github.com/openclaw/openclaw) gateway running
-- OpenAI API key with Realtime API access
+- An OpenAI-compatible local transcription endpoint
+- A local Chatterbox endpoint or ElevenLabs API credentials
## 🚀 Installation
@@ -203,6 +204,34 @@ cd clawbody
/venvs/apps_venv/bin/pip install -e .
```
+### Native Reachy App configuration
+
+The native app reads deployment environment variables at startup. A packaged
+`.env` file is optional for development only; environment variables supplied by
+the app manager take precedence. Configure at least:
+
+```text
+OPENCLAW_GATEWAY_URL=ws://your-openclaw-gateway:18789
+OPENCLAW_TOKEN=your-gateway-token
+OPENCLAW_AGENT_ID=main
+OPENCLAW_SESSION_KEY=reachy
+STT_BASE_URL=http://your-speech-host:8890/v1
+STT_MODEL=distil-large-v3
+CHATTERBOX_URL=http://your-speech-host:8890/v1/audio/speech
+CHATTERBOX_VOICE=asmo
+ENABLE_WAKE_PHRASE=true
+WAKE_PHRASE=Hey Asmo
+ENABLE_ATTENTION_QUEUE=true
+ATTENTION_API_URL=http://192.168.1.238:18790
+ATTENTION_POLL_SECONDS=60
+```
+
+The native app uses the SDK-owned robot connection and targets the local Reachy
+daemon for camera tracking. It does not require a laptop-side console or a
+second OpenClaw agent. Keep gateway, STT, TTS, and optional ElevenLabs secrets
+in the app manager's secret/environment store; never commit them to `.env`, a
+Space, or the repository.
+
## ⚙️ Configuration
1. Copy the example environment file:
@@ -214,22 +243,35 @@ cp .env.example .env
2. Edit `.env` with your configuration:
```bash
-# Required
-OPENAI_API_KEY=sk-...your-key...
+# Required local STT
+STT_BASE_URL=http://speech-host.local:8890/v1
+STT_MODEL=distil-large-v3
# OpenClaw Gateway (required for AI responses)
OPENCLAW_GATEWAY_URL=http://localhost:18789 # or your host IP
OPENCLAW_TOKEN=your-gateway-token
OPENCLAW_AGENT_ID=main
-# Optional - Customize voice
-OPENAI_VOICE=cedar
+# Local Chatterbox voice profile
+TTS_PROVIDER=chatterbox
+CHATTERBOX_URL=http://speech-host.local:8890/v1/audio/speech
+CHATTERBOX_VOICE=default
# Optional - Face tracking (enabled by default)
ENABLE_FACE_TRACKING=true
HEAD_TRACKER_TYPE=mediapipe # or "yolo" for more accuracy
+
+# Optional - require commands to begin with "Hey Asmo"
+ENABLE_WAKE_PHRASE=true
+WAKE_PHRASE=Hey Asmo
```
+Wake phrase matching is case-insensitive and tolerates STT punctuation such as
+`Hey, Asmo`. It must occur at the start of each utterance. The phrase is removed
+before the command is sent to OpenClaw; saying only the phrase is treated as a
+greeting. Leave `ENABLE_WAKE_PHRASE=false` for the original always-responsive
+behavior.
+
## 🎮 Usage
### With Simulator
@@ -308,7 +350,7 @@ ClawBody builds on:
- [Pollen Robotics](https://www.pollen-robotics.com/) - Reachy Mini robot, SDK, and simulator
- [OpenClaw](https://github.com/openclaw/openclaw) - AI assistant framework (Clawson!)
-- [OpenAI](https://openai.com/) - Realtime API for voice I/O
+- [Resemble AI Chatterbox](https://github.com/resemble-ai/chatterbox) - local speech synthesis
- [MuJoCo](https://mujoco.org/) - Physics simulation engine
- [pollen-robotics/reachy_mini_conversation_app](https://huggingface.co/spaces/pollen-robotics/reachy_mini_conversation_app) - Movement and audio systems
diff --git a/index.html b/index.html
index 79d1c84..5069592 100644
--- a/index.html
+++ b/index.html
@@ -26,15 +26,15 @@
Give OpenClaw a physical body.
Connect your OpenClaw AI assistant (Clawson) to a Reachy Mini robot.
- Ultra-responsive voice conversation through OpenAI Realtime API,
- intelligent responses from OpenClaw, and expressive robot movements.
+ Local speech recognition and expressive synthesis, intelligent
+ responses from OpenClaw, and expressive robot movements.
-
🎙️ OpenAI Realtime
+
🎙️ Local STT
🦞 OpenClaw Gateway
💃 Expressive movement
🖥️ No robot required!
@@ -128,9 +128,9 @@
How it works
- 🎤 Robot (or simulator) captures your voice
- 🔇 Voice Activity Detection identifies when you stop speaking
- - 📝 OpenAI Realtime transcribes your speech instantly
+ - 📝 Always-hot local STT transcribes your speech
- 🦞 OpenClaw processes your message with full AI capabilities
- - 🔊 OpenAI Realtime speaks the response naturally
+ - 🔊 Chatterbox or ElevenLabs speaks the response naturally
- 🤖 Robot moves expressively while speaking
diff --git a/pyproject.toml b/pyproject.toml
index 5ee2495..a1763ec 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,23 +3,24 @@ requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"
[project]
-name = "clawbody"
-version = "0.1.0"
-description = "ClawBody - Give your OpenClaw AI agent a physical robot body with Reachy Mini. Voice conversation powered by OpenAI Realtime API with expressive movements."
+name = "clawbodylocal"
+version = "0.2.1"
+description = "Asmo's local OpenClaw body for Reachy Mini, with local STT, pluggable TTS, and expressive movement."
readme = "README.md"
license = {text = "Apache-2.0"}
requires-python = ">=3.11"
authors = [
- {name = "Tom", email = "tom@example.com"}
+ {name = "Asmo", email = "stanme@asmo.bot"}
]
keywords = [
+ "clawbodylocal",
"clawbody",
"reachy-mini",
"openclaw",
"clawson",
"robotics",
"ai-assistant",
- "openai-realtime",
+ "local-speech",
"voice-conversation",
"expressive-robot",
"embodied-ai"
@@ -35,13 +36,20 @@ classifiers = [
"Topic :: Scientific/Engineering :: Human Machine Interfaces",
]
dependencies = [
- # OpenAI Realtime API
+ # Optional OpenAI client retained for the cloud vision fallback
"openai>=1.50.0",
+
+ # Local speech provider clients
+ "httpx>=0.27.0",
- # Audio streaming
- "fastrtc>=0.0.17",
+ # Audio processing
"numpy",
"scipy",
+ "pillow>=10",
+ # Native apps share Reachy's apps venv. Pin the validated SDK in the main
+ # dependency set so installing the dance library cannot downgrade it.
+ "reachy-mini==1.9.0",
+ "reachy-mini-dances-library>=0.2.1",
# OpenClaw gateway client (WebSocket protocol)
"websockets>=12.0",
@@ -53,11 +61,10 @@ dependencies = [
"python-dotenv",
]
-# Note: reachy-mini SDK must be installed separately from the robot or GitHub:
-# pip install git+https://github.com/pollen-robotics/reachy_mini.git
-# Or on the robot, it's pre-installed.
-
[project.optional-dependencies]
+robot = [
+ "reachy-mini==1.9.0",
+]
wireless = [
"pygobject",
]
@@ -92,16 +99,18 @@ dev = [
]
[project.scripts]
+clawbodylocal = "reachy_mini_openclaw.main:main"
+# Compatibility alias for existing local launchers and operator muscle memory.
clawbody = "reachy_mini_openclaw.main:main"
[project.entry-points."reachy_mini_apps"]
-clawbody = "reachy_mini_openclaw.main:ClawBodyApp"
+clawbodylocal = "reachy_mini_openclaw.main:ReachyMiniOpenclaw"
[project.urls]
-Homepage = "https://github.com/yourusername/clawbody"
-Documentation = "https://github.com/yourusername/clawbody#readme"
-Repository = "https://github.com/yourusername/clawbody"
-Issues = "https://github.com/yourusername/clawbody/issues"
+Homepage = "https://huggingface.co/spaces/asmolebot/clawbodylocal"
+Documentation = "https://github.com/asmolebot/clawbody#readme"
+Repository = "https://github.com/asmolebot/clawbody"
+Issues = "https://github.com/asmolebot/clawbody/issues"
[tool.setuptools.packages.find]
where = ["src"]
@@ -114,6 +123,9 @@ target-version = "py311"
select = ["E", "F", "I", "N", "W", "UP"]
ignore = ["E501"]
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
[tool.mypy]
python_version = "3.11"
warn_return_any = true
diff --git a/services/voice_stack/README.md b/services/voice_stack/README.md
new file mode 100644
index 0000000..289c01d
--- /dev/null
+++ b/services/voice_stack/README.md
@@ -0,0 +1,14 @@
+# Xeon voice stack
+
+Always-hot English speech services for ClawBody:
+
+- `POST /v1/audio/transcriptions`: faster-whisper `distil-large-v3`
+- `POST /v1/audio/speech`: Chatterbox Turbo cloned from a configured voice reference
+- `GET /health`: loaded-model readiness
+
+The service intentionally exposes an OpenAI-shaped API contract so ClawBody can switch providers without changing its conversation loop. It binds to loopback by default; use the Reachy LAN address only after firewalling it to the robot subnet.
+
+```bash
+uv sync
+ASMO_VOICE_REFERENCE=/path/to/asmo.wav uv run uvicorn app:app --host 127.0.0.1 --port 8890
+```
diff --git a/services/voice_stack/app.py b/services/voice_stack/app.py
new file mode 100644
index 0000000..fd0fe59
--- /dev/null
+++ b/services/voice_stack/app.py
@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import io
+import os
+import tempfile
+import threading
+import time
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Annotated
+
+import soundfile as sf
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.responses import Response
+from pydantic import BaseModel, Field
+
+
+class SpeechRequest(BaseModel):
+ input: str = Field(min_length=1, max_length=1200)
+ voice: str = "asmo"
+ response_format: str = "wav"
+ speed: float = Field(default=1.0, ge=0.75, le=1.25)
+ temperature: float = Field(default=0.8, ge=0.05, le=2.0)
+ repetition_penalty: float = Field(default=1.2, ge=1.0, le=2.0)
+
+
+class ModelRegistry:
+ def __init__(self) -> None:
+ self.device = os.getenv("VOICE_DEVICE", "cuda")
+ self.stt_model_name = os.getenv("STT_MODEL", "distil-large-v3")
+ self.stt_compute_type = os.getenv("STT_COMPUTE_TYPE", "int8_float16")
+ self.voice_reference = Path(os.environ["ASMO_VOICE_REFERENCE"])
+ self.stt = None
+ self.tts = None
+ self.stt_lock = threading.Lock()
+ self.tts_lock = threading.Lock()
+ self.loaded_at: float | None = None
+
+ def load(self) -> None:
+ if not self.voice_reference.is_file():
+ raise RuntimeError(f"Voice reference does not exist: {self.voice_reference}")
+
+ from faster_whisper import WhisperModel
+ from chatterbox.tts_turbo import ChatterboxTurboTTS
+
+ self.stt = WhisperModel(
+ self.stt_model_name,
+ device=self.device,
+ compute_type=self.stt_compute_type,
+ )
+ self.tts = ChatterboxTurboTTS.from_pretrained(self.device)
+ self.loaded_at = time.time()
+
+ @property
+ def ready(self) -> bool:
+ return self.stt is not None and self.tts is not None
+
+
+models: ModelRegistry | None = None
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+ global models
+ models = ModelRegistry()
+ models.load()
+ yield
+
+
+app = FastAPI(title="ClawBody Voice Stack", version="0.1.0", lifespan=lifespan)
+
+
+def registry() -> ModelRegistry:
+ if models is None or not models.ready:
+ raise HTTPException(status_code=503, detail="Models are not ready")
+ return models
+
+
+@app.get("/health")
+def health() -> dict[str, object]:
+ current = registry()
+ return {
+ "ok": True,
+ "device": current.device,
+ "stt_model": current.stt_model_name,
+ "tts_model": "ResembleAI/chatterbox-turbo",
+ "loaded_at": current.loaded_at,
+ }
+
+
+@app.post("/v1/audio/transcriptions")
+def transcribe(
+ file: Annotated[UploadFile, File()],
+ language: Annotated[str, Form()] = "en",
+ response_format: Annotated[str, Form()] = "json",
+) -> dict[str, object]:
+ current = registry()
+ suffix = Path(file.filename or "utterance.wav").suffix or ".wav"
+ with tempfile.NamedTemporaryFile(suffix=suffix) as tmp:
+ while chunk := file.file.read(1024 * 1024):
+ tmp.write(chunk)
+ tmp.flush()
+ started = time.perf_counter()
+ with current.stt_lock:
+ segments, info = current.stt.transcribe(
+ tmp.name,
+ language=language,
+ beam_size=1,
+ vad_filter=True,
+ condition_on_previous_text=False,
+ )
+ segment_list = list(segments)
+
+ text = " ".join(segment.text.strip() for segment in segment_list).strip()
+ elapsed = time.perf_counter() - started
+ result = {
+ "text": text,
+ "language": info.language,
+ "duration": info.duration,
+ "elapsed_seconds": elapsed,
+ }
+ if response_format == "verbose_json":
+ result["segments"] = [
+ {"start": segment.start, "end": segment.end, "text": segment.text.strip()}
+ for segment in segment_list
+ ]
+ return result
+
+
+@app.post("/v1/audio/speech")
+def synthesize(request: SpeechRequest) -> Response:
+ if request.response_format != "wav":
+ raise HTTPException(status_code=400, detail="Only wav output is supported locally")
+ if request.voice != "asmo":
+ raise HTTPException(status_code=400, detail="Unknown local voice")
+
+ current = registry()
+ started = time.perf_counter()
+ with current.tts_lock:
+ waveform = current.tts.generate(
+ request.input,
+ audio_prompt_path=str(current.voice_reference),
+ temperature=request.temperature,
+ repetition_penalty=request.repetition_penalty,
+ norm_loudness=True,
+ )
+
+ audio = waveform.squeeze(0).detach().cpu().numpy()
+ output = io.BytesIO()
+ sf.write(output, audio, current.tts.sr, format="WAV", subtype="PCM_16")
+ return Response(
+ content=output.getvalue(),
+ media_type="audio/wav",
+ headers={
+ "X-Generation-Seconds": f"{time.perf_counter() - started:.3f}",
+ "X-Voice-Model": "chatterbox-turbo",
+ },
+ )
diff --git a/services/voice_stack/pyproject.toml b/services/voice_stack/pyproject.toml
new file mode 100644
index 0000000..825a13a
--- /dev/null
+++ b/services/voice_stack/pyproject.toml
@@ -0,0 +1,28 @@
+[project]
+name = "asmo-voice-stack"
+version = "0.1.0"
+description = "Always-hot local STT and voice-cloned TTS for ClawBody"
+requires-python = ">=3.11,<3.12"
+dependencies = [
+ "chatterbox-tts>=0.1.6",
+ "fastapi>=0.116.0",
+ "faster-whisper>=1.2.0",
+ "numpy<2",
+ "python-multipart>=0.0.20",
+ "setuptools<81",
+ "soundfile>=0.13.1",
+ "uvicorn[standard]>=0.35.0",
+]
+
+[dependency-groups]
+dev = [
+ "httpx>=0.28.1",
+ "pytest>=8.4.0",
+]
+
+[tool.uv]
+package = false
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]
diff --git a/services/voice_stack/tests/test_contract.py b/services/voice_stack/tests/test_contract.py
new file mode 100644
index 0000000..def1218
--- /dev/null
+++ b/services/voice_stack/tests/test_contract.py
@@ -0,0 +1,33 @@
+import os
+
+os.environ.setdefault("ASMO_VOICE_REFERENCE", __file__)
+
+from fastapi.testclient import TestClient
+
+import app as voice_app
+
+
+class FakeRegistry:
+ ready = True
+ device = "cuda"
+ stt_model_name = "distil-large-v3"
+ loaded_at = 1.0
+
+
+def test_health_contract(monkeypatch):
+ monkeypatch.setattr(voice_app, "models", FakeRegistry())
+ client = TestClient(voice_app.app)
+ response = client.get("/health")
+ assert response.status_code == 200
+ assert response.json()["ok"] is True
+ assert response.json()["stt_model"] == "distil-large-v3"
+
+
+def test_speech_rejects_unknown_voice(monkeypatch):
+ monkeypatch.setattr(voice_app, "models", FakeRegistry())
+ client = TestClient(voice_app.app)
+ response = client.post(
+ "/v1/audio/speech",
+ json={"input": "hello", "voice": "somebody-else"},
+ )
+ assert response.status_code == 400
diff --git a/src/reachy_mini_openclaw/attention.py b/src/reachy_mini_openclaw/attention.py
new file mode 100644
index 0000000..095faa3
--- /dev/null
+++ b/src/reachy_mini_openclaw/attention.py
@@ -0,0 +1,176 @@
+"""Quiet, consent-driven attention notifications for ClawBody Local."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+import time
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TapDetector:
+ """Detect a deliberate double tap from Reachy's head IMU."""
+
+ acceleration_delta: float = 2.4
+ gyroscope_threshold: float = 0.55
+ min_gap: float = 0.12
+ max_gap: float = 0.85
+ refractory: float = 1.5
+ baseline_alpha: float = 0.04
+ _baseline: float | None = None
+ _first_impact_at: float | None = None
+ _last_impact_at: float = -100.0
+ _refractory_until: float = 0.0
+
+ def feed(self, imu: dict[str, Any] | None, now: float | None = None) -> bool:
+ if not imu:
+ return False
+ now = time.monotonic() if now is None else now
+ acceleration = imu.get("accelerometer") or []
+ gyroscope = imu.get("gyroscope") or []
+ if len(acceleration) != 3 or len(gyroscope) != 3:
+ return False
+ accel_norm = math.sqrt(sum(float(value) ** 2 for value in acceleration))
+ gyro_norm = math.sqrt(sum(float(value) ** 2 for value in gyroscope))
+ if self._baseline is None:
+ self._baseline = accel_norm
+ return False
+ delta = abs(accel_norm - self._baseline)
+ if delta < self.acceleration_delta:
+ self._baseline = (
+ (1.0 - self.baseline_alpha) * self._baseline
+ + self.baseline_alpha * accel_norm
+ )
+ impact = delta >= self.acceleration_delta or gyro_norm >= self.gyroscope_threshold
+ if not impact or now < self._refractory_until:
+ if self._first_impact_at is not None and now - self._first_impact_at > self.max_gap:
+ self._first_impact_at = None
+ return False
+ if now - self._last_impact_at < self.min_gap:
+ return False
+ self._last_impact_at = now
+ if self._first_impact_at is None:
+ self._first_impact_at = now
+ return False
+ gap = now - self._first_impact_at
+ self._first_impact_at = None
+ if self.min_gap <= gap <= self.max_gap:
+ self._refractory_until = now + self.refractory
+ return True
+ self._first_impact_at = now
+ return False
+
+
+class AttentionClient:
+ def __init__(self, base_url: str, token: str, timeout: float = 8.0) -> None:
+ self._http = httpx.AsyncClient(
+ base_url=base_url.rstrip("/"),
+ headers={"Authorization": f"Bearer {token}"},
+ timeout=timeout,
+ )
+
+ async def pending(self) -> dict[str, Any]:
+ response = await self._http.get("/v1/pending")
+ response.raise_for_status()
+ payload = response.json()
+ return payload if isinstance(payload, dict) else {}
+
+ async def acknowledge(self, event_ids: list[str]) -> None:
+ response = await self._http.post("/v1/ack", json={"eventIds": event_ids})
+ response.raise_for_status()
+
+ async def close(self) -> None:
+ await self._http.aclose()
+
+
+def format_attention(snapshot: dict[str, Any]) -> str:
+ needs_attention = snapshot.get("needsAttention") or []
+ actions_taken = snapshot.get("actionsTaken") or []
+ parts: list[str] = []
+ if needs_attention:
+ parts.append("Here's what needs your attention.")
+ for item in needs_attention[:6]:
+ source = "Gmail" if item.get("source") == "gmail" else "Annexus"
+ sender = str(item.get("sender") or "unknown sender")
+ subject = str(item.get("subject") or "no subject")
+ parts.append(f"{source}, from {sender}: {subject}.")
+ remaining = len(needs_attention) - 6
+ if remaining > 0:
+ parts.append(f"There {'is' if remaining == 1 else 'are'} {remaining} more item{'s' if remaining != 1 else ''}.")
+ if actions_taken:
+ parts.append("Actions taken.")
+ parts.extend(str(action) for action in actions_taken[:6])
+ remaining = len(actions_taken) - 6
+ if remaining > 0:
+ parts.append(f"There {'is' if remaining == 1 else 'are'} {remaining} more action{'s' if remaining != 1 else ''}.")
+ return " ".join(parts) or "There is nothing waiting for your attention."
+
+
+class AttentionController:
+ def __init__(
+ self,
+ *,
+ robot: Any,
+ movement_manager: Any,
+ handler: Any,
+ client: AttentionClient,
+ poll_seconds: float = 60.0,
+ tap_detector: TapDetector | None = None,
+ ) -> None:
+ self.robot = robot
+ self.movement_manager = movement_manager
+ self.handler = handler
+ self.client = client
+ self.poll_seconds = max(5.0, poll_seconds)
+ self.tap_detector = tap_detector or TapDetector()
+ self._snapshot: dict[str, Any] = {}
+ self._last_poll = -self.poll_seconds
+ self._stopping = False
+
+ async def run(self) -> None:
+ try:
+ while not self._stopping:
+ now = time.monotonic()
+ if now - self._last_poll >= self.poll_seconds:
+ self._last_poll = now
+ try:
+ self._snapshot = await self.client.pending()
+ self.movement_manager.set_attention_pending(
+ bool(self._snapshot.get("hasPending"))
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ logger.warning("Attention queue poll failed: %s", exc)
+
+ if self._snapshot.get("hasPending"):
+ status = self.movement_manager.get_status()
+ can_accept_tap = not status.get("is_listening") and not status.get("processing")
+ if can_accept_tap and self.tap_detector.feed(self.robot.imu, now):
+ await self._read_pending()
+ await asyncio.sleep(0.05 if self._snapshot.get("hasPending") else 0.5)
+ finally:
+ self.movement_manager.set_attention_pending(False)
+ await self.client.close()
+
+ async def _read_pending(self) -> None:
+ event_ids = [str(value) for value in self._snapshot.get("eventIds") or []]
+ if not event_ids:
+ return
+ logger.info("Attention double-tap detected; reading %d queued event(s)", len(event_ids))
+ text = format_attention(self._snapshot)
+ duration = await self.handler.speak_text(text)
+ await asyncio.sleep(duration + 0.4)
+ await self.client.acknowledge(event_ids)
+ self._snapshot = {}
+ self.movement_manager.set_attention_pending(False)
+
+ async def stop(self) -> None:
+ self._stopping = True
diff --git a/src/reachy_mini_openclaw/audio/conversation.py b/src/reachy_mini_openclaw/audio/conversation.py
new file mode 100644
index 0000000..243dd7f
--- /dev/null
+++ b/src/reachy_mini_openclaw/audio/conversation.py
@@ -0,0 +1,430 @@
+"""Provider-based voice pipeline with OpenClaw as the only conversational brain."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+from difflib import SequenceMatcher
+import logging
+import re
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+
+from reachy_mini_openclaw.audio.providers import SpeechToText, TextToSpeech
+
+logger = logging.getLogger(__name__)
+
+_VISION_REQUESTS = (
+ re.compile(r"\b(?:can|do) you see\b", re.I),
+ re.compile(r"\bwhat (?:do|can) you see\b", re.I),
+ re.compile(r"\blook at\b", re.I),
+ re.compile(r"\bwhat(?:'s| is) (?:this|that|in front of you|on my|on the)\b", re.I),
+ re.compile(r"\bwho (?:is|are) (?:here|there|in front of you)\b", re.I),
+)
+
+
+def needs_vision(text: str) -> bool:
+ return any(pattern.search(text) for pattern in _VISION_REQUESTS)
+
+
+def command_after_wake_phrase(text: str, wake_phrase: str) -> str | None:
+ """Return the command after a spoken wake phrase, or None if absent.
+
+ Matching is case-insensitive, restricted to the start of the transcript,
+ and tolerant of punctuation inserted by STT (for example, "Hey, Asmo").
+ """
+ words = re.findall(r"[^\W_]+", wake_phrase, flags=re.UNICODE)
+ if not words:
+ return None
+ spoken = list(re.finditer(r"[^\W_]+", text, flags=re.UNICODE))
+ if len(spoken) < len(words):
+ return None
+
+ for expected, observed_match in zip(words, spoken[: len(words)], strict=True):
+ observed = observed_match.group(0)
+ expected_folded = expected.casefold()
+ observed_folded = observed.casefold()
+ if observed_folded == expected_folded:
+ continue
+ # Short wake words are occasionally normalized by Whisper ("hey" ->
+ # "hi"). Names are even more vulnerable ("Asmo" -> "asthma").
+ # Keep the fuzzy allowance narrow: it only applies to the leading wake
+ # phrase tokens and still requires the entire configured phrase.
+ if expected_folded == "hey" and observed_folded == "hi":
+ continue
+ threshold = 0.60 if len(expected_folded) >= 4 else 0.75
+ if (
+ abs(len(expected_folded) - len(observed_folded)) > 2
+ or SequenceMatcher(None, expected_folded, observed_folded).ratio() < threshold
+ ):
+ return None
+
+ return re.sub(
+ r"^[\s,;:!?.\-–—]+", "", text[spoken[len(words) - 1].end() :]
+ ).strip()
+
+
+@dataclass(frozen=True)
+class Utterance:
+ sample_rate: int
+ samples: NDArray[np.float32]
+
+
+class EnergyVAD:
+ """Small deterministic utterance segmenter for the always-hot local STT path."""
+
+ def __init__(
+ self,
+ *,
+ rms_threshold: float = 0.022,
+ activation_ms: int = 160,
+ prefix_ms: int = 250,
+ silence_ms: int = 650,
+ min_speech_ms: int = 300,
+ max_speech_seconds: float = 20.0,
+ ) -> None:
+ self.rms_threshold = rms_threshold
+ self.activation_ms = activation_ms
+ self.prefix_ms = prefix_ms
+ self.silence_ms = silence_ms
+ self.min_speech_ms = min_speech_ms
+ self.max_speech_seconds = max_speech_seconds
+ self.speaking = False
+ self._sample_rate = 0
+ self._prefix: deque[NDArray[np.float32]] = deque()
+ self._prefix_samples = 0
+ self._frames: list[NDArray[np.float32]] = []
+ self._activation_samples = 0
+ self._voiced_samples = 0
+ self._silence_samples = 0
+
+ @staticmethod
+ def normalize(audio: NDArray[Any]) -> NDArray[np.float32]:
+ values = np.asarray(audio)
+ if values.ndim == 2:
+ channel_axis = 0 if values.shape[0] <= values.shape[1] else 1
+ values = values.mean(axis=channel_axis)
+ values = values.reshape(-1)
+ if values.dtype == np.int16:
+ return values.astype(np.float32) / 32768.0
+ return values.astype(np.float32)
+
+ def feed(
+ self,
+ sample_rate: int,
+ audio: NDArray[Any],
+ *,
+ allow_start: bool = True,
+ ) -> Utterance | None:
+ frame = self.normalize(audio)
+ if frame.size == 0:
+ return None
+ if self._sample_rate and sample_rate != self._sample_rate:
+ self.reset()
+ self._sample_rate = sample_rate
+ rms = float(np.sqrt(np.mean(np.square(frame, dtype=np.float32))))
+ voiced = rms >= self.rms_threshold and (self.speaking or allow_start)
+
+ if not self.speaking:
+ self._prefix.append(frame.copy())
+ self._prefix_samples += frame.size
+ prefix_limit = max(1, int(sample_rate * self.prefix_ms / 1000))
+ while self._prefix and self._prefix_samples - self._prefix[0].size >= prefix_limit:
+ self._prefix_samples -= self._prefix.popleft().size
+ if not voiced:
+ self._activation_samples = 0
+ return None
+ self._activation_samples += frame.size
+ activation_limit = max(1, int(sample_rate * self.activation_ms / 1000))
+ if self._activation_samples < activation_limit:
+ return None
+ self.speaking = True
+ self._frames = list(self._prefix)
+ self._voiced_samples = self._activation_samples
+ self._silence_samples = 0
+ self._prefix.clear()
+ self._prefix_samples = 0
+ return None
+
+ self._frames.append(frame.copy())
+ if voiced:
+ self._voiced_samples += frame.size
+ self._silence_samples = 0
+ else:
+ self._silence_samples += frame.size
+
+ silence_limit = int(sample_rate * self.silence_ms / 1000)
+ max_limit = int(sample_rate * self.max_speech_seconds)
+ if self._silence_samples < silence_limit and sum(chunk.size for chunk in self._frames) < max_limit:
+ return None
+
+ min_voiced = int(sample_rate * self.min_speech_ms / 1000)
+ samples = np.concatenate(self._frames) if self._voiced_samples >= min_voiced else None
+ self.reset()
+ return Utterance(sample_rate, samples) if samples is not None else None
+
+ def reset(self) -> None:
+ self.speaking = False
+ self._sample_rate = 0
+ self._prefix.clear()
+ self._prefix_samples = 0
+ self._frames = []
+ self._activation_samples = 0
+ self._voiced_samples = 0
+ self._silence_samples = 0
+
+
+class ConversationHandler:
+ """Turn Reachy microphone frames into spoken responses from OpenClaw."""
+
+ def __init__(
+ self,
+ *,
+ stt: SpeechToText,
+ tts: TextToSpeech,
+ openclaw_bridge: Any,
+ deps: Any,
+ vad: EnergyVAD,
+ output_chunk_samples: int = 2048,
+ attention_provider: Any = None,
+ require_hardware_speech: bool = True,
+ wake_phrase_enabled: bool = False,
+ wake_phrase: str = "Hey Asmo",
+ ) -> None:
+ self.stt = stt
+ self.tts = tts
+ self.openclaw_bridge = openclaw_bridge
+ self.deps = deps
+ self.vad = vad
+ self.output_chunk_samples = output_chunk_samples
+ self.attention_provider = attention_provider
+ self.require_hardware_speech = require_hardware_speech
+ self.wake_phrase_enabled = wake_phrase_enabled
+ self.wake_phrase = wake_phrase
+ self.output_queue: asyncio.Queue[tuple[int, NDArray[np.int16]] | dict[str, str]] = asyncio.Queue()
+ self._utterances: asyncio.Queue[Utterance] = asyncio.Queue(maxsize=2)
+ self._shutdown_requested = False
+ self._ignore_input_until = 0.0
+ self._turn_lock = asyncio.Lock()
+
+ async def _retry_async(self, operation: Any, label: str, attempts: int = 3) -> Any:
+ """Retry a transient network/provider operation with bounded backoff."""
+ delay = 0.25
+ for attempt in range(1, attempts + 1):
+ try:
+ return await operation()
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ if attempt == attempts:
+ raise
+ logger.warning(
+ "%s unavailable; retrying in %.2fs (%d/%d)",
+ label,
+ delay,
+ attempt,
+ attempts - 1,
+ )
+ await asyncio.sleep(delay)
+ delay = min(delay * 2, 2.0)
+ raise AssertionError("retry loop exhausted")
+
+ async def _ensure_gateway(self) -> None:
+ """Reconnect the gateway without making a failed turn fatal immediately."""
+ if self.openclaw_bridge.is_connected:
+ return
+
+ async def connect() -> bool:
+ if await self.openclaw_bridge.connect():
+ return True
+ raise RuntimeError("OpenClaw gateway is unavailable")
+
+ await self._retry_async(connect, "OpenClaw gateway")
+
+ async def start_up(self) -> None:
+ while not self._shutdown_requested:
+ utterance = await self._utterances.get()
+ try:
+ await self._process_utterance(utterance)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ logger.error("Voice turn failed: %s", exc, exc_info=True)
+ self.deps.movement_manager.set_processing(False)
+
+ async def receive(self, frame: tuple[int, NDArray[Any]]) -> None:
+ sample_rate, audio = frame
+ if time.monotonic() < self._ignore_input_until:
+ if self.vad.speaking:
+ self.vad.reset()
+ self.deps.movement_manager.set_listening(False)
+ return
+ was_speaking = self.vad.speaking
+ allow_start = True
+ attention_state = None
+ if not was_speaking and self.require_hardware_speech and self.attention_provider is not None:
+ attention_state = self.attention_provider.get_attention_state()
+ # Fail open if DoA is unavailable. When it is available, require
+ # the microphone array's own speech classifier as a second vote.
+ if attention_state.get("doa_angle") is not None:
+ allow_start = bool(attention_state.get("speech_detected"))
+ utterance = self.vad.feed(sample_rate, audio, allow_start=allow_start)
+ if self.vad.speaking and not was_speaking:
+ logger.info("Listening started (attention=%s)", attention_state)
+ self.deps.movement_manager.set_listening(True)
+ self.deps.movement_manager.set_processing(False)
+ self._clear_audio_output()
+ if self.deps.head_wobbler is not None:
+ self.deps.head_wobbler.reset()
+ if was_speaking and not self.vad.speaking:
+ self.deps.movement_manager.set_listening(False)
+ if utterance is not None:
+ if self._utterances.full():
+ logger.warning("Dropping utterance because the conversational pipeline is busy")
+ return
+ await self._utterances.put(utterance)
+
+ async def _process_utterance(self, utterance: Utterance) -> None:
+ async with self._turn_lock:
+ await self._process_utterance_locked(utterance)
+
+ async def _process_utterance_locked(self, utterance: Utterance) -> None:
+ turn_started = time.monotonic()
+ self.deps.movement_manager.set_processing(True)
+ transcript = await self._retry_async(
+ lambda: self.stt.transcribe(utterance.samples, utterance.sample_rate),
+ "STT service",
+ )
+ stt_done = time.monotonic()
+ if not transcript:
+ self.deps.movement_manager.set_processing(False)
+ return
+ transcript = transcript.strip()
+ if self.wake_phrase_enabled:
+ command = command_after_wake_phrase(transcript, self.wake_phrase)
+ if command is None:
+ logger.info("Ignored utterance without configured wake phrase: %r", transcript)
+ self.deps.movement_manager.set_processing(False)
+ return
+ # A wake phrase by itself remains a valid greeting. When a command
+ # follows it, keep the conversational session free of boilerplate.
+ transcript = command or transcript
+ logger.info("User: %s", transcript)
+ await self.output_queue.put({"role": "user", "content": transcript})
+
+ await self._ensure_gateway()
+ motion_catalog = getattr(self.deps, "motion_catalog", None)
+ motion_hint = f" {motion_catalog.response_hint()}" if motion_catalog is not None else ""
+ image_b64 = None
+ if needs_vision(transcript) and self.attention_provider is not None:
+ jpeg = self.attention_provider.get_latest_jpeg()
+ if jpeg:
+ image_b64 = base64.b64encode(jpeg).decode("ascii")
+ logger.info("Attached current Reachy camera frame to visual question")
+ response = await self.openclaw_bridge.chat(
+ transcript,
+ image_b64=image_b64,
+ system_context=(
+ "The user is speaking to you through your Reachy Mini body. "
+ "Answer naturally and concisely for spoken delivery. "
+ "When a camera image is attached, it is your current first-person view."
+ + motion_hint
+ ),
+ )
+ openclaw_done = time.monotonic()
+ if response.error:
+ raise RuntimeError(f"OpenClaw response failed: {response.error}")
+ if motion_catalog is not None:
+ text, directive = motion_catalog.parse_response(response.content)
+ directive = (
+ directive
+ or motion_catalog.infer_request(transcript)
+ or motion_catalog.infer(text)
+ )
+ else:
+ text, directive = response.content.strip(), None
+ if not text:
+ raise RuntimeError("OpenClaw returned an empty response")
+
+ logger.info("Assistant: %s", text)
+ audio = await self._retry_async(lambda: self.tts.synthesize(text), "TTS service")
+ tts_done = time.monotonic()
+ # Reachy's microphone hears its own speaker over the remote WebRTC
+ # stream. Suppress VAD for the synthesized duration plus a short room-
+ # echo tail so the robot does not recursively answer itself.
+ self._ignore_input_until = max(
+ self._ignore_input_until,
+ time.monotonic() + (audio.samples.size / audio.sample_rate) + 0.8,
+ )
+ self.vad.reset()
+ self.deps.movement_manager.set_listening(False)
+ self.deps.movement_manager.set_processing(False)
+ # Start primary motion only after synthesis is ready. This synchronizes
+ # the gesture with playback and prevents thinking offsets from being
+ # blended into the opening of recorded choreography.
+ if directive is not None:
+ logger.info("Motion: %s:%s", directive.kind, directive.name)
+ self.deps.movement_manager.queue_move(
+ motion_catalog.get(directive.kind, directive.name)
+ )
+ await self.output_queue.put({"role": "assistant", "content": text})
+ for offset in range(0, audio.samples.size, self.output_chunk_samples):
+ chunk = audio.samples[offset : offset + self.output_chunk_samples]
+ await self.output_queue.put((audio.sample_rate, chunk.reshape(1, -1)))
+ logger.info(
+ "Turn latency: stt=%.2fs openclaw=%.2fs tts=%.2fs ready=%.2fs",
+ stt_done - turn_started,
+ openclaw_done - stt_done,
+ tts_done - openclaw_done,
+ tts_done - turn_started,
+ )
+
+ async def speak_text(self, text: str) -> float:
+ """Speak trusted local notification text without creating an agent turn."""
+ async with self._turn_lock:
+ self.deps.movement_manager.set_processing(False)
+ audio = await self._retry_async(lambda: self.tts.synthesize(text), "TTS service")
+ duration = audio.samples.size / audio.sample_rate
+ self._ignore_input_until = max(
+ self._ignore_input_until,
+ time.monotonic() + duration + 0.8,
+ )
+ self.vad.reset()
+ self.deps.movement_manager.set_listening(False)
+ await self.output_queue.put({"role": "assistant", "content": text})
+ for offset in range(0, audio.samples.size, self.output_chunk_samples):
+ chunk = audio.samples[offset : offset + self.output_chunk_samples]
+ await self.output_queue.put((audio.sample_rate, chunk.reshape(1, -1)))
+ return duration
+
+ def _clear_audio_output(self) -> None:
+ retained: list[dict[str, str]] = []
+ while not self.output_queue.empty():
+ try:
+ item = self.output_queue.get_nowait()
+ if isinstance(item, dict):
+ retained.append(item)
+ except asyncio.QueueEmpty:
+ break
+ for item in retained:
+ self.output_queue.put_nowait(item)
+
+ async def emit(self) -> tuple[int, NDArray[np.int16]] | dict[str, str] | None:
+ try:
+ return await asyncio.wait_for(self.output_queue.get(), timeout=0.1)
+ except TimeoutError:
+ return None
+
+ async def shutdown(self) -> None:
+ self._shutdown_requested = True
+ self.vad.reset()
+ self._clear_audio_output()
+ self.deps.movement_manager.set_listening(False)
+ self.deps.movement_manager.set_processing(False)
+ if self.deps.head_wobbler is not None:
+ self.deps.head_wobbler.reset()
diff --git a/src/reachy_mini_openclaw/audio/playback.py b/src/reachy_mini_openclaw/audio/playback.py
new file mode 100644
index 0000000..62738b3
--- /dev/null
+++ b/src/reachy_mini_openclaw/audio/playback.py
@@ -0,0 +1,12 @@
+"""Audio preparation helpers for Reachy speaker playback."""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+def apply_output_gain(samples: NDArray[np.floating], gain: float) -> NDArray[np.float32]:
+ """Apply software gain and safely constrain samples to Reachy's float range."""
+ audio = np.asarray(samples, dtype=np.float32)
+ return np.clip(audio * gain, -1.0, 1.0).astype(np.float32, copy=False)
diff --git a/src/reachy_mini_openclaw/audio/providers.py b/src/reachy_mini_openclaw/audio/providers.py
new file mode 100644
index 0000000..81dbe08
--- /dev/null
+++ b/src/reachy_mini_openclaw/audio/providers.py
@@ -0,0 +1,204 @@
+"""Provider adapters for ClawBody speech recognition and synthesis."""
+
+from __future__ import annotations
+
+import io
+import logging
+import wave
+from dataclasses import dataclass
+from typing import Protocol
+
+import httpx
+import numpy as np
+from numpy.typing import NDArray
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class SynthesizedAudio:
+ """Mono signed 16-bit PCM returned by a TTS provider."""
+
+ sample_rate: int
+ samples: NDArray[np.int16]
+
+
+class SpeechToText(Protocol):
+ async def transcribe(self, samples: NDArray[np.float32], sample_rate: int) -> str: ...
+
+
+class TextToSpeech(Protocol):
+ async def synthesize(self, text: str) -> SynthesizedAudio: ...
+
+
+def encode_wav(samples: NDArray[np.float32], sample_rate: int) -> bytes:
+ """Encode normalized mono float audio as a PCM16 WAV."""
+ mono = np.asarray(samples, dtype=np.float32).reshape(-1)
+ pcm = (np.clip(mono, -1.0, 1.0) * 32767.0).astype(" SynthesizedAudio:
+ """Decode a mono/stereo PCM WAV response to mono int16 samples."""
+ with wave.open(io.BytesIO(payload), "rb") as wav:
+ channels = wav.getnchannels()
+ width = wav.getsampwidth()
+ sample_rate = wav.getframerate()
+ frames = wav.readframes(wav.getnframes())
+ if width != 2:
+ raise ValueError(f"TTS WAV must be PCM16, got {width * 8}-bit audio")
+ samples = np.frombuffer(frames, dtype=" 1:
+ samples = samples.reshape(-1, channels).astype(np.int32).mean(axis=1).astype(np.int16)
+ return SynthesizedAudio(sample_rate=sample_rate, samples=samples.copy())
+
+
+class OpenAICompatibleSTT:
+ """Client for a local/OpenAI-compatible audio transcription endpoint."""
+
+ def __init__(
+ self,
+ base_url: str,
+ model: str,
+ *,
+ api_key: str | None = None,
+ language: str = "en",
+ timeout: float = 30.0,
+ client: httpx.AsyncClient | None = None,
+ ) -> None:
+ base = base_url.rstrip("/")
+ self.url = base if base.endswith("/audio/transcriptions") else f"{base}/audio/transcriptions"
+ self.model = model
+ self.api_key = api_key
+ self.language = language
+ self.timeout = timeout
+ self._client = client
+
+ async def transcribe(self, samples: NDArray[np.float32], sample_rate: int) -> str:
+ headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+ data = {"model": self.model, "response_format": "json"}
+ if self.language:
+ data["language"] = self.language
+ files = {"file": ("utterance.wav", encode_wav(samples, sample_rate), "audio/wav")}
+ client = self._client or httpx.AsyncClient(timeout=self.timeout)
+ try:
+ response = await client.post(self.url, headers=headers, data=data, files=files)
+ response.raise_for_status()
+ body = response.json()
+ text = body.get("text", "") if isinstance(body, dict) else ""
+ return str(text).strip()
+ finally:
+ if self._client is None:
+ await client.aclose()
+
+
+class ChatterboxTTS:
+ """Client for the local Chatterbox OpenAI-compatible speech endpoint."""
+
+ def __init__(
+ self,
+ url: str,
+ *,
+ voice: str = "asmo",
+ api_key: str | None = None,
+ sample_rate: int = 24000,
+ timeout: float = 45.0,
+ client: httpx.AsyncClient | None = None,
+ ) -> None:
+ self.url = url
+ self.voice = voice
+ self.api_key = api_key
+ self.sample_rate = sample_rate
+ self.timeout = timeout
+ self._client = client
+
+ async def synthesize(self, text: str) -> SynthesizedAudio:
+ headers = {"Accept": "audio/wav"}
+ if self.api_key:
+ headers["Authorization"] = f"Bearer {self.api_key}"
+ payload = {
+ "input": text,
+ "voice": self.voice,
+ "model": "chatterbox-turbo",
+ "response_format": "wav",
+ }
+ client = self._client or httpx.AsyncClient(timeout=self.timeout)
+ try:
+ response = await client.post(self.url, headers=headers, json=payload)
+ response.raise_for_status()
+ content_type = response.headers.get("content-type", "").lower()
+ if "wav" in content_type or response.content[:4] == b"RIFF":
+ return decode_wav(response.content)
+ return SynthesizedAudio(
+ sample_rate=self.sample_rate,
+ samples=np.frombuffer(response.content, dtype=" None:
+ self.url = f"{base_url.rstrip('/')}/v1/text-to-speech/{voice_id}/stream"
+ self.api_key = api_key
+ self.model_id = model_id
+ self.sample_rate = sample_rate
+ self.timeout = timeout
+ self._client = client
+
+ async def synthesize(self, text: str) -> SynthesizedAudio:
+ client = self._client or httpx.AsyncClient(timeout=self.timeout)
+ try:
+ response = await client.post(
+ self.url,
+ params={"output_format": f"pcm_{self.sample_rate}"},
+ headers={"xi-api-key": self.api_key, "Accept": "application/octet-stream"},
+ json={"text": text, "model_id": self.model_id},
+ )
+ response.raise_for_status()
+ return SynthesizedAudio(
+ sample_rate=self.sample_rate,
+ samples=np.frombuffer(response.content, dtype=" None:
+ if not providers:
+ raise ValueError("at least one TTS provider is required")
+ self.providers = providers
+
+ async def synthesize(self, text: str) -> SynthesizedAudio:
+ last_error: Exception | None = None
+ for provider in self.providers:
+ try:
+ return await provider.synthesize(text)
+ except Exception as exc:
+ last_error = exc
+ logger.warning("TTS provider %s failed: %s", type(provider).__name__, exc)
+ assert last_error is not None
+ raise last_error
diff --git a/src/reachy_mini_openclaw/camera_worker.py b/src/reachy_mini_openclaw/camera_worker.py
index 453b50d..36570be 100644
--- a/src/reachy_mini_openclaw/camera_worker.py
+++ b/src/reachy_mini_openclaw/camera_worker.py
@@ -13,9 +13,13 @@
import time
import logging
import threading
+import math
+from io import BytesIO
from typing import Any, List, Tuple, Optional
+import httpx
import numpy as np
+from PIL import Image
from numpy.typing import NDArray
from scipy.spatial.transform import Rotation as R
@@ -36,7 +40,16 @@ class CameraWorker:
RETURNING -- interpolating back to neutral before scanning again
"""
- def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None:
+ def __init__(
+ self,
+ reachy_mini: ReachyMini,
+ head_tracker: Any = None,
+ *,
+ daemon_tracking: bool = False,
+ daemon_url: str | None = None,
+ tracking_weight: float = 0.7,
+ sound_tracking: bool = True,
+ ) -> None:
"""Initialize camera worker.
Args:
@@ -45,6 +58,11 @@ def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None:
"""
self.reachy_mini = reachy_mini
self.head_tracker = head_tracker
+ self.daemon_tracking = daemon_tracking
+ self.daemon_url = daemon_url.rstrip("/") if daemon_url else None
+ self.tracking_weight = min(1.0, max(0.0, tracking_weight))
+ self.sound_tracking = sound_tracking
+ self._http = httpx.Client(timeout=0.5) if self.daemon_url else None
# Thread-safe frame storage
self.latest_frame: Optional[NDArray[np.uint8]] = None
@@ -58,6 +76,12 @@ def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None:
0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
] # x, y, z, roll, pitch, yaw
self.face_tracking_lock = threading.Lock()
+ self.face_detected = False
+ self.speech_detected = False
+ self.doa_angle: Optional[float] = None
+ self._last_attention_poll = 0.0
+ self._sound_yaw = 0.0
+ self._ever_received_frame = False
# Face tracking timing (for smooth interpolation back to neutral)
self.last_face_detected_time: Optional[float] = None
@@ -104,6 +128,15 @@ def get_latest_frame(self) -> Optional[NDArray[np.uint8]]:
return None
return self.latest_frame.copy()
+ def get_latest_jpeg(self, quality: int = 80) -> bytes | None:
+ """Encode the latest BGR camera frame for an OpenClaw image turn."""
+ frame = self.get_latest_frame()
+ if frame is None:
+ return None
+ output = BytesIO()
+ Image.fromarray(frame[:, :, ::-1]).save(output, format="JPEG", quality=quality)
+ return output.getvalue()
+
def get_face_tracking_offsets(
self,
) -> Tuple[float, float, float, float, float, float]:
@@ -128,6 +161,11 @@ def set_head_tracking_enabled(self, enabled: bool) -> None:
# Start scanning immediately when re-enabled
self._start_scanning()
self.is_head_tracking_enabled = enabled
+ if self.daemon_tracking:
+ if enabled:
+ self.reachy_mini.start_head_tracking(weight=self.tracking_weight)
+ else:
+ self.reachy_mini.stop_head_tracking()
logger.info("Head tracking %s", "enabled" if enabled else "disabled")
def start(self) -> None:
@@ -142,8 +180,67 @@ def stop(self) -> None:
self._stop_event.set()
if self._thread is not None:
self._thread.join(timeout=2.0)
+ if self.daemon_tracking:
+ try:
+ self.reachy_mini.stop_head_tracking()
+ except Exception:
+ logger.debug("Could not stop daemon head tracking", exc_info=True)
+ if self._http is not None:
+ self._http.close()
logger.info("Camera worker stopped")
+ def get_attention_state(self) -> dict[str, Any]:
+ """Return the latest daemon face and microphone-array state."""
+ with self.face_tracking_lock:
+ return {
+ "face_detected": self.face_detected,
+ "speech_detected": self.speech_detected,
+ "doa_angle": self.doa_angle,
+ }
+
+ def _update_daemon_attention(self, current_time: float) -> None:
+ """Blend daemon face tracking with a conservative sound-direction cue."""
+ if current_time - self._last_attention_poll < 0.1:
+ return
+ self._last_attention_poll = current_time
+
+ try:
+ face = self.reachy_mini.get_tracked_face(wait=False)
+ face_detected = bool(face and face.detected)
+ except Exception:
+ face_detected = False
+
+ doa_angle: Optional[float] = None
+ speech_detected = False
+ if self.sound_tracking and self._http is not None:
+ try:
+ payload = self._http.get(f"{self.daemon_url}/api/state/doa").json()
+ if payload:
+ doa_angle = float(payload["angle"])
+ speech_detected = bool(payload["speech_detected"])
+ except Exception:
+ logger.debug("DoA poll failed", exc_info=True)
+
+ # Visual tracking runs inside the daemon and wins whenever a face is
+ # visible. Otherwise, softly bias the normal app pose toward speech.
+ if face_detected:
+ target_yaw = 0.0
+ elif speech_detected and doa_angle is not None:
+ target_yaw = float(np.clip((math.pi / 2) - doa_angle, -0.75, 0.75))
+ else:
+ target_yaw = 0.0
+
+ alpha = 0.35 if speech_detected and not face_detected else 0.18
+ self._sound_yaw = alpha * target_yaw + (1.0 - alpha) * self._sound_yaw
+ if abs(self._sound_yaw) < 0.01:
+ self._sound_yaw = 0.0
+
+ with self.face_tracking_lock:
+ self.face_detected = face_detected
+ self.speech_detected = speech_detected
+ self.doa_angle = doa_angle
+ self.face_tracking_offsets = [0.0, 0.0, 0.0, 0.0, 0.0, self._sound_yaw]
+
# ------------------------------------------------------------------
# Scanning helpers
# ------------------------------------------------------------------
@@ -202,10 +299,16 @@ def _working_loop(self) -> None:
frame = self.reachy_mini.media.get_frame()
if frame is not None:
+ if not self._ever_received_frame:
+ self._ever_received_frame = True
+ logger.info("First camera frame received: shape=%s", frame.shape)
# Thread-safe frame storage
with self.frame_lock:
self.latest_frame = frame
+ if self.daemon_tracking and self.is_head_tracking_enabled:
+ self._update_daemon_attention(current_time)
+
# Check if face tracking was just disabled
if self.previous_head_tracking_state and not self.is_head_tracking_enabled:
# Face tracking was just disabled - start interpolation to neutral
@@ -218,7 +321,9 @@ def _working_loop(self) -> None:
self.previous_head_tracking_state = self.is_head_tracking_enabled
# Handle face tracking if enabled and head tracker available
- if self.is_head_tracking_enabled and self.head_tracker is not None:
+ if self.daemon_tracking:
+ pass
+ elif self.is_head_tracking_enabled and self.head_tracker is not None:
self._process_face_tracking(frame, current_time, neutral_pose)
elif self.last_face_detected_time is not None:
# Handle interpolation back to neutral when tracking disabled
@@ -379,4 +484,4 @@ def _interpolate_to_neutral(
self.interpolation_start_time = None
self.interpolation_start_pose = None
self._smoothed_offsets = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
- self._start_scanning()
\ No newline at end of file
+ self._start_scanning()
diff --git a/src/reachy_mini_openclaw/config.py b/src/reachy_mini_openclaw/config.py
index 0061f45..1370648 100644
--- a/src/reachy_mini_openclaw/config.py
+++ b/src/reachy_mini_openclaw/config.py
@@ -3,6 +3,7 @@
Handles environment variables and configuration settings for the application.
"""
+import json
import os
from pathlib import Path
from dataclasses import dataclass, field
@@ -15,25 +16,85 @@
load_dotenv(_project_root / ".env")
+def _gateway_token() -> Optional[str]:
+ token = os.getenv("OPENCLAW_TOKEN") or os.getenv("OPENCLAW_GATEWAY_TOKEN")
+ if token:
+ return token
+ config_path = Path.home() / ".openclaw" / "openclaw.json"
+ try:
+ data = json.loads(config_path.read_text(encoding="utf-8"))
+ return data.get("gateway", {}).get("auth", {}).get("token")
+ except (OSError, ValueError, TypeError):
+ return None
+
+
@dataclass
class Config:
"""Application configuration loaded from environment variables."""
- # OpenAI Configuration
+ # Optional OpenAI configuration (currently used only by the legacy vision fallback)
OPENAI_API_KEY: str = field(default_factory=lambda: os.getenv("OPENAI_API_KEY", ""))
- OPENAI_MODEL: str = field(default_factory=lambda: os.getenv("OPENAI_MODEL", "gpt-realtime-1.5"))
- OPENAI_VOICE: str = field(default_factory=lambda: os.getenv("OPENAI_VOICE", "cedar"))
+
+ # Speech-to-text. The endpoint follows OpenAI's multipart
+ # /v1/audio/transcriptions contract, but can be entirely local.
+ STT_BASE_URL: str = field(default_factory=lambda: os.getenv("STT_BASE_URL", ""))
+ STT_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("STT_API_KEY"))
+ STT_MODEL: str = field(default_factory=lambda: os.getenv("STT_MODEL", "distil-large-v3"))
+ STT_LANGUAGE: str = field(default_factory=lambda: os.getenv("STT_LANGUAGE", "en"))
+ STT_TIMEOUT_SECONDS: float = field(default_factory=lambda: float(os.getenv("STT_TIMEOUT_SECONDS", "30")))
+
+ # Text-to-speech. Chatterbox is primary; ElevenLabs can be enabled as a
+ # fallback without changing the conversational pipeline.
+ TTS_PROVIDER: str = field(default_factory=lambda: os.getenv("TTS_PROVIDER", "chatterbox").lower())
+ CHATTERBOX_URL: str = field(default_factory=lambda: os.getenv("CHATTERBOX_URL", ""))
+ CHATTERBOX_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("CHATTERBOX_API_KEY"))
+ CHATTERBOX_VOICE: str = field(default_factory=lambda: os.getenv("CHATTERBOX_VOICE", "asmo"))
+ CHATTERBOX_SAMPLE_RATE: int = field(default_factory=lambda: int(os.getenv("CHATTERBOX_SAMPLE_RATE", "24000")))
+ ELEVENLABS_API_KEY: Optional[str] = field(default_factory=lambda: os.getenv("ELEVENLABS_API_KEY"))
+ ELEVENLABS_VOICE_ID: Optional[str] = field(default_factory=lambda: os.getenv("ELEVENLABS_VOICE_ID"))
+ ELEVENLABS_MODEL_ID: str = field(default_factory=lambda: os.getenv("ELEVENLABS_MODEL_ID", "eleven_flash_v2_5"))
+ ELEVENLABS_BASE_URL: str = field(default_factory=lambda: os.getenv("ELEVENLABS_BASE_URL", "https://api.elevenlabs.io"))
+ TTS_TIMEOUT_SECONDS: float = field(default_factory=lambda: float(os.getenv("TTS_TIMEOUT_SECONDS", "45")))
+ # Software gain applied immediately before Reachy playback. Reachy's app
+ # volume remains the final hardware control; 1.0 preserves the TTS signal.
+ AUDIO_OUTPUT_GAIN: float = field(default_factory=lambda: float(os.getenv("AUDIO_OUTPUT_GAIN", "1.0")))
+
+ # Local voice activity detection.
+ VAD_RMS_THRESHOLD: float = field(default_factory=lambda: float(os.getenv("VAD_RMS_THRESHOLD", "0.022")))
+ VAD_ACTIVATION_MS: int = field(default_factory=lambda: int(os.getenv("VAD_ACTIVATION_MS", "160")))
+ VAD_PREFIX_MS: int = field(default_factory=lambda: int(os.getenv("VAD_PREFIX_MS", "250")))
+ VAD_SILENCE_MS: int = field(default_factory=lambda: int(os.getenv("VAD_SILENCE_MS", "650")))
+ VAD_MIN_SPEECH_MS: int = field(default_factory=lambda: int(os.getenv("VAD_MIN_SPEECH_MS", "300")))
+ VAD_MAX_SPEECH_SECONDS: float = field(default_factory=lambda: float(os.getenv("VAD_MAX_SPEECH_SECONDS", "20")))
+ VAD_REQUIRE_HARDWARE_SPEECH: bool = field(default_factory=lambda: os.getenv("VAD_REQUIRE_HARDWARE_SPEECH", "true").lower() == "true")
+
+ # Optional transcript-level wake phrase gate. Audio is still transcribed
+ # locally, but only utterances beginning with the phrase reach OpenClaw.
+ ENABLE_WAKE_PHRASE: bool = field(default_factory=lambda: os.getenv("ENABLE_WAKE_PHRASE", "false").lower() == "true")
+ WAKE_PHRASE: str = field(default_factory=lambda: os.getenv("WAKE_PHRASE", "Hey Asmo"))
+
+ # Hourly mail attention queue. The broker is gateway-local and uses the
+ # same bearer token already provisioned for OpenClaw.
+ ENABLE_ATTENTION_QUEUE: bool = field(default_factory=lambda: os.getenv("ENABLE_ATTENTION_QUEUE", "true").lower() == "true")
+ ATTENTION_API_URL: str = field(default_factory=lambda: os.getenv("ATTENTION_API_URL", "http://192.168.1.238:18790"))
+ ATTENTION_POLL_SECONDS: float = field(default_factory=lambda: float(os.getenv("ATTENTION_POLL_SECONDS", "60")))
# OpenClaw Gateway Configuration
- OPENCLAW_GATEWAY_URL: str = field(default_factory=lambda: os.getenv("OPENCLAW_GATEWAY_URL", "ws://localhost:18789"))
- OPENCLAW_TOKEN: Optional[str] = field(default_factory=lambda: os.getenv("OPENCLAW_TOKEN"))
+ OPENCLAW_GATEWAY_URL: str = field(default_factory=lambda: os.getenv("OPENCLAW_GATEWAY_URL", ""))
+ OPENCLAW_TOKEN: Optional[str] = field(default_factory=_gateway_token)
OPENCLAW_AGENT_ID: str = field(default_factory=lambda: os.getenv("OPENCLAW_AGENT_ID", "main"))
+ # Spoken body turns should feel immediate. "off" is OpenClaw's lowest-
+ # latency setting and applies only to the dedicated Reachy session.
+ OPENCLAW_THINKING: str = field(default_factory=lambda: os.getenv("OPENCLAW_THINKING", "off").lower())
# Session key for OpenClaw - uses "main" to share context with WhatsApp and other channels
# Format: agent::, but we only need the session key part here
OPENCLAW_SESSION_KEY: str = field(default_factory=lambda: os.getenv("OPENCLAW_SESSION_KEY", "main"))
# Robot Configuration
ROBOT_NAME: Optional[str] = field(default_factory=lambda: os.getenv("ROBOT_NAME"))
+ ROBOT_HOST: str = field(default_factory=lambda: os.getenv("ROBOT_HOST", "reachy-mini.local"))
+ ROBOT_PORT: int = field(default_factory=lambda: int(os.getenv("ROBOT_PORT", "8000")))
+ ROBOT_CONNECTION_MODE: str = field(default_factory=lambda: os.getenv("ROBOT_CONNECTION_MODE", "auto"))
# Feature Flags
ENABLE_OPENCLAW_TOOLS: bool = field(default_factory=lambda: os.getenv("ENABLE_OPENCLAW_TOOLS", "true").lower() == "true")
@@ -41,8 +102,11 @@ class Config:
ENABLE_FACE_TRACKING: bool = field(default_factory=lambda: os.getenv("ENABLE_FACE_TRACKING", "true").lower() == "true")
# Face Tracking Configuration
- # Options: "yolo", "mediapipe", or None for auto-detect
- HEAD_TRACKER_TYPE: Optional[str] = field(default_factory=lambda: os.getenv("HEAD_TRACKER_TYPE", "yolo"))
+ # "daemon" uses Reachy 1.9's built-in detector and avoids a duplicate model.
+ # "yolo" and "mediapipe" remain available for older daemon versions.
+ HEAD_TRACKER_TYPE: Optional[str] = field(default_factory=lambda: os.getenv("HEAD_TRACKER_TYPE", "daemon"))
+ FACE_TRACKING_WEIGHT: float = field(default_factory=lambda: float(os.getenv("FACE_TRACKING_WEIGHT", "0.7")))
+ ENABLE_SOUND_TRACKING: bool = field(default_factory=lambda: os.getenv("ENABLE_SOUND_TRACKING", "true").lower() == "true")
# Local Vision Processing
ENABLE_LOCAL_VISION: bool = field(default_factory=lambda: os.getenv("ENABLE_LOCAL_VISION", "false").lower() == "true")
@@ -53,11 +117,36 @@ class Config:
# Custom Profile (for personality customization)
CUSTOM_PROFILE: Optional[str] = field(default_factory=lambda: os.getenv("REACHY_MINI_CUSTOM_PROFILE"))
- def validate(self) -> list[str]:
+ def validate(self, gateway_url: Optional[str] = None) -> list[str]:
"""Validate configuration and return list of errors."""
errors = []
- if not self.OPENAI_API_KEY:
- errors.append("OPENAI_API_KEY is required")
+ if not self.STT_BASE_URL:
+ errors.append("STT_BASE_URL is required")
+ if not (gateway_url or self.OPENCLAW_GATEWAY_URL):
+ errors.append("OPENCLAW_GATEWAY_URL is required")
+ if self.TTS_PROVIDER not in {"chatterbox", "elevenlabs"}:
+ errors.append("TTS_PROVIDER must be 'chatterbox' or 'elevenlabs'")
+ if self.TTS_PROVIDER == "chatterbox" and not self.CHATTERBOX_URL:
+ errors.append("CHATTERBOX_URL is required for Chatterbox TTS")
+ if self.TTS_PROVIDER == "elevenlabs":
+ if not self.ELEVENLABS_API_KEY:
+ errors.append("ELEVENLABS_API_KEY is required for ElevenLabs TTS")
+ if not self.ELEVENLABS_VOICE_ID:
+ errors.append("ELEVENLABS_VOICE_ID is required for ElevenLabs TTS")
+ if self.AUDIO_OUTPUT_GAIN <= 0:
+ errors.append("AUDIO_OUTPUT_GAIN must be greater than zero")
+ if self.ENABLE_WAKE_PHRASE and not self.WAKE_PHRASE.strip():
+ errors.append("WAKE_PHRASE must not be empty when wake phrase gating is enabled")
+ if self.ENABLE_ATTENTION_QUEUE and not self.ATTENTION_API_URL:
+ errors.append("ATTENTION_API_URL is required when attention queue is enabled")
+ if self.ENABLE_ATTENTION_QUEUE and not self.OPENCLAW_TOKEN:
+ errors.append("OPENCLAW_TOKEN is required when attention queue is enabled")
+ if self.ATTENTION_POLL_SECONDS < 5:
+ errors.append("ATTENTION_POLL_SECONDS must be at least 5")
+ if self.OPENCLAW_THINKING not in {
+ "off", "minimal", "low", "medium", "high", "xhigh", "adaptive", "max"
+ }:
+ errors.append("OPENCLAW_THINKING is not a supported thinking level")
return errors
diff --git a/src/reachy_mini_openclaw/gradio_app.py b/src/reachy_mini_openclaw/gradio_app.py
index 4afc4fd..5bbc1fc 100644
--- a/src/reachy_mini_openclaw/gradio_app.py
+++ b/src/reachy_mini_openclaw/gradio_app.py
@@ -7,8 +7,9 @@
- Manual control options
"""
-import os
+import asyncio
import logging
+import threading
from typing import Optional
import gradio as gr
@@ -17,8 +18,11 @@
def launch_gradio(
- gateway_url: str = "ws://localhost:18789",
+ gateway_url: str = "",
robot_name: Optional[str] = None,
+ robot_host: Optional[str] = None,
+ robot_port: Optional[int] = None,
+ robot_connection_mode: Optional[str] = None,
enable_camera: bool = True,
enable_openclaw: bool = True,
enable_face_tracking: bool = True,
@@ -30,6 +34,9 @@ def launch_gradio(
Args:
gateway_url: OpenClaw gateway URL
robot_name: Robot name for connection
+ robot_host: Reachy daemon hostname or IP
+ robot_port: Reachy daemon TCP port
+ robot_connection_mode: Reachy SDK connection mode
enable_camera: Whether to enable camera
enable_openclaw: Whether to enable OpenClaw
enable_face_tracking: Whether to enable face tracking
@@ -41,40 +48,52 @@ def launch_gradio(
# State
app_instance = None
+ app_loop = None
def start_conversation():
"""Start the conversation."""
- nonlocal app_instance
-
+ nonlocal app_instance, app_loop
+
from reachy_mini_openclaw.main import ClawBodyCore
- import asyncio
- import threading
if app_instance is not None:
return "Already running"
try:
+ config.ENABLE_FACE_TRACKING = enable_face_tracking
+ if head_tracker_type is not None:
+ config.HEAD_TRACKER_TYPE = head_tracker_type
app_instance = ClawBodyCore(
gateway_url=gateway_url,
robot_name=robot_name,
+ robot_host=robot_host,
+ robot_port=robot_port,
+ robot_connection_mode=robot_connection_mode,
enable_camera=enable_camera,
enable_openclaw=enable_openclaw,
- enable_face_tracking=enable_face_tracking,
- head_tracker_type=head_tracker_type,
)
# Run in background thread
- def run_app():
+ def run_app(instance):
+ nonlocal app_instance, app_loop
loop = asyncio.new_event_loop()
+ app_loop = loop
asyncio.set_event_loop(loop)
try:
- loop.run_until_complete(app_instance.run())
+ loop.run_until_complete(instance.run())
+ except asyncio.CancelledError:
+ logger.info("App stopped")
except Exception as e:
logger.error("App error: %s", e)
finally:
+ loop.run_until_complete(instance.stop())
+ if app_instance is instance:
+ app_instance = None
+ app_loop = None
+ asyncio.set_event_loop(None)
loop.close()
- thread = threading.Thread(target=run_app, daemon=True)
+ thread = threading.Thread(target=run_app, args=(app_instance,), daemon=True)
thread.start()
return "Started successfully"
@@ -83,13 +102,16 @@ def run_app():
def stop_conversation():
"""Stop the conversation."""
- nonlocal app_instance
+ nonlocal app_instance, app_loop
if app_instance is None:
return "Not running"
try:
- app_instance.stop()
+ if app_loop is None or not app_loop.is_running():
+ return "App is still starting"
+ future = asyncio.run_coroutine_threadsafe(app_instance.stop(), app_loop)
+ future.result(timeout=10)
app_instance = None
return "Stopped"
except Exception as e:
@@ -112,7 +134,7 @@ def save_profile(name, instructions):
# 🤖 Reachy Mini OpenClaw
Give your OpenClaw AI agent a physical presence with Reachy Mini.
- Using OpenAI Realtime API for responsive voice conversation.
+ OpenClaw is the brain, with local STT and pluggable speech synthesis.
""")
with gr.Tab("Conversation"):
@@ -164,8 +186,10 @@ def save_profile(name, instructions):
### Current Configuration
- **OpenClaw Gateway**: {gateway_url}
- - **OpenAI Model**: {config.OPENAI_MODEL}
- - **Voice**: {config.OPENAI_VOICE}
+ - **STT Endpoint**: {config.STT_BASE_URL}
+ - **STT Model**: {config.STT_MODEL}
+ - **TTS Provider**: {config.TTS_PROVIDER}
+ - **TTS Voice**: {config.CHATTERBOX_VOICE if config.TTS_PROVIDER == 'chatterbox' else config.ELEVENLABS_VOICE_ID}
- **Camera Enabled**: {enable_camera}
- **OpenClaw Enabled**: {enable_openclaw}
- **Face Tracking**: {enable_face_tracking}
@@ -180,8 +204,9 @@ def save_profile(name, instructions):
This application combines:
- - **OpenAI Realtime API** for ultra-low-latency voice conversation
- - **OpenClaw Gateway** for extended AI capabilities (web, calendar, smart home, etc.)
+ - **Local speech recognition** through an OpenAI-compatible endpoint
+ - **OpenClaw Gateway** as the conversational brain and tool runtime
+ - **Chatterbox or ElevenLabs** for speech synthesis
- **Reachy Mini Robot** for physical embodiment with expressive movements
### Features
@@ -196,7 +221,7 @@ def save_profile(name, instructions):
- [Reachy Mini SDK](https://github.com/pollen-robotics/reachy_mini)
- [OpenClaw](https://github.com/openclaw/openclaw)
- - [OpenAI Realtime API](https://platform.openai.com/docs/guides/realtime)
+ - [Chatterbox](https://github.com/resemble-ai/chatterbox)
""")
demo.launch(share=share, server_name="0.0.0.0", server_port=7860)
diff --git a/src/reachy_mini_openclaw/main.py b/src/reachy_mini_openclaw/main.py
index 98a6dc2..2a74756 100644
--- a/src/reachy_mini_openclaw/main.py
+++ b/src/reachy_mini_openclaw/main.py
@@ -1,8 +1,8 @@
"""ClawBody - Give your OpenClaw AI agent a physical robot body.
This module provides the main application that connects:
-- OpenAI Realtime API for voice I/O (speech recognition + TTS)
-- OpenClaw Gateway for AI intelligence (Clawson's brain)
+- Local/OpenAI-compatible STT and provider-based TTS
+- OpenClaw Gateway as the sole conversational brain
- Reachy Mini robot for physical embodiment
Usage:
@@ -17,9 +17,9 @@
"""
import os
-import sys
import time
import asyncio
+import base64
import logging
import argparse
import threading
@@ -27,10 +27,11 @@
from typing import Any, Optional
from dotenv import load_dotenv
+from reachy_mini import ReachyMiniApp
-# Load environment from project root (override=True ensures .env takes precedence)
+# Load local development settings without overriding deployment-injected env.
_project_root = Path(__file__).parent.parent.parent
-load_dotenv(_project_root / ".env", override=True)
+load_dotenv(_project_root / ".env", override=False)
logger = logging.getLogger(__name__)
@@ -76,7 +77,7 @@ def parse_args() -> argparse.Namespace:
clawbody --robot-name my-reachy
# Use different OpenClaw gateway
- clawbody --gateway-url http://192.168.1.100:18790
+ clawbody --gateway-url http://openclaw-host.local:18789
"""
)
@@ -95,11 +96,29 @@ def parse_args() -> argparse.Namespace:
type=str,
help="Robot name for connection (default: auto-discover)"
)
+ parser.add_argument(
+ "--robot-host",
+ type=str,
+ default=os.getenv("ROBOT_HOST", "reachy-mini.local"),
+ help="Reachy daemon hostname or IP",
+ )
+ parser.add_argument(
+ "--robot-port",
+ type=int,
+ default=int(os.getenv("ROBOT_PORT", "8000")),
+ help="Reachy daemon TCP port",
+ )
+ parser.add_argument(
+ "--robot-connection-mode",
+ choices=["auto", "localhost_only", "network"],
+ default=os.getenv("ROBOT_CONNECTION_MODE", "auto"),
+ help="Reachy SDK connection mode",
+ )
parser.add_argument(
"--gateway-url",
type=str,
- default=os.getenv("OPENCLAW_GATEWAY_URL", "ws://localhost:18789"),
- help="OpenClaw gateway URL (from OPENCLAW_GATEWAY_URL env or default)"
+ default=os.getenv("OPENCLAW_GATEWAY_URL", ""),
+ help="OpenClaw gateway URL (required unless supplied by environment)"
)
parser.add_argument(
"--no-camera",
@@ -135,15 +154,18 @@ class ClawBodyCore:
This class orchestrates all components:
- Reachy Mini robot connection and movement control
- - OpenAI Realtime API for voice I/O
- - OpenClaw gateway bridge for AI intelligence
+ - Provider-based speech recognition and synthesis
+ - OpenClaw gateway bridge for conversational intelligence
- Audio input/output loops
"""
def __init__(
self,
- gateway_url: str = "ws://localhost:18789",
+ gateway_url: str = "",
robot_name: Optional[str] = None,
+ robot_host: Optional[str] = None,
+ robot_port: Optional[int] = None,
+ robot_connection_mode: Optional[str] = None,
enable_camera: bool = True,
enable_openclaw: bool = True,
robot: Optional["ReachyMini"] = None,
@@ -154,6 +176,9 @@ def __init__(
Args:
gateway_url: OpenClaw gateway URL
robot_name: Optional robot name for connection
+ robot_host: Reachy daemon hostname or IP
+ robot_port: Reachy daemon TCP port
+ robot_connection_mode: Reachy SDK connection mode
enable_camera: Whether to enable camera functionality
enable_openclaw: Whether to enable OpenClaw integration
robot: Optional pre-initialized robot (for app framework)
@@ -164,19 +189,27 @@ def __init__(
from reachy_mini_openclaw.moves import MovementManager
from reachy_mini_openclaw.audio.head_wobbler import HeadWobbler
from reachy_mini_openclaw.openclaw_bridge import OpenClawBridge
+ from reachy_mini_openclaw.motion_catalog import MotionCatalog
from reachy_mini_openclaw.tools.core_tools import ToolDependencies
- from reachy_mini_openclaw.openai_realtime import OpenAIRealtimeHandler
+ from reachy_mini_openclaw.audio.conversation import ConversationHandler, EnergyVAD
+ from reachy_mini_openclaw.audio.providers import (
+ ChatterboxTTS,
+ ElevenLabsTTS,
+ FallbackTTS,
+ OpenAICompatibleSTT,
+ )
self.gateway_url = gateway_url
+ self.audio_output_gain = config.AUDIO_OUTPUT_GAIN
self._external_stop_event = external_stop_event
self._owns_robot = robot is None
# Validate configuration
- errors = config.validate()
+ errors = config.validate(gateway_url=self.gateway_url)
if errors:
for error in errors:
logger.error("Config error: %s", error)
- sys.exit(1)
+ raise ValueError("Invalid ClawBody configuration")
# Connect to robot
if robot is not None:
@@ -187,16 +220,19 @@ def __init__(
robot_kwargs = {}
if robot_name:
robot_kwargs["robot_name"] = robot_name
+ robot_kwargs["host"] = robot_host or config.ROBOT_HOST
+ robot_kwargs["port"] = robot_port or config.ROBOT_PORT
+ robot_kwargs["connection_mode"] = robot_connection_mode or config.ROBOT_CONNECTION_MODE
try:
self.robot = ReachyMini(**robot_kwargs)
except TimeoutError as e:
logger.error("Connection timeout: %s", e)
logger.error("Check that the robot is powered on and reachable.")
- sys.exit(1)
+ raise RuntimeError("Reachy Mini connection timed out") from e
except Exception as e:
logger.error("Robot connection failed: %s", e)
- sys.exit(1)
+ raise RuntimeError("Reachy Mini connection failed") from e
logger.info("Connected to robot: %s", self.robot.client.get_status())
@@ -206,6 +242,13 @@ def __init__(
self.head_wobbler = HeadWobbler(
set_speech_offsets=self.movement_manager.set_speech_offsets
)
+ logger.info("Loading Reachy emotion and dance catalogs...")
+ self.motion_catalog = MotionCatalog()
+ logger.info(
+ "Loaded %d emotion moves and %d dance moves",
+ len(self.motion_catalog.list("emotion")),
+ len(self.motion_catalog.list("dance")),
+ )
# Initialize OpenClaw bridge
self.openclaw_bridge = None
@@ -224,19 +267,29 @@ def __init__(
if enable_camera:
logger.info("Initializing camera worker...")
from reachy_mini_openclaw.camera_worker import CameraWorker
+
+ daemon_tracking = config.ENABLE_FACE_TRACKING and config.HEAD_TRACKER_TYPE == "daemon"
# Initialize head tracker for local face tracking
- if config.ENABLE_FACE_TRACKING:
+ if config.ENABLE_FACE_TRACKING and not daemon_tracking:
self.head_tracker = self._initialize_head_tracker(config.HEAD_TRACKER_TYPE)
# Initialize camera worker with head tracker
self.camera_worker = CameraWorker(
reachy_mini=self.robot,
head_tracker=self.head_tracker,
+ daemon_tracking=daemon_tracking,
+ daemon_url=(
+ "http://127.0.0.1:8000"
+ if robot is not None
+ else f"http://{robot_host or config.ROBOT_HOST}:{robot_port or config.ROBOT_PORT}"
+ ),
+ tracking_weight=config.FACE_TRACKING_WEIGHT,
+ sound_tracking=config.ENABLE_SOUND_TRACKING,
)
# Enable/disable head tracking based on whether we have a tracker
- self.camera_worker.set_head_tracking_enabled(self.head_tracker is not None)
+ self.camera_worker.set_head_tracking_enabled(daemon_tracking or self.head_tracker is not None)
# Initialize local vision processor if enabled
if config.ENABLE_LOCAL_VISION:
@@ -250,13 +303,76 @@ def __init__(
camera_worker=self.camera_worker,
openclaw_bridge=self.openclaw_bridge,
vision_manager=self.vision_manager,
+ motion_catalog=self.motion_catalog,
)
- # Initialize OpenAI Realtime handler with OpenClaw bridge
- self.handler = OpenAIRealtimeHandler(
+ if self.openclaw_bridge is None:
+ raise ValueError("The conversational pipeline requires OpenClaw")
+
+ stt = OpenAICompatibleSTT(
+ config.STT_BASE_URL,
+ config.STT_MODEL,
+ api_key=config.STT_API_KEY,
+ language=config.STT_LANGUAGE,
+ timeout=config.STT_TIMEOUT_SECONDS,
+ )
+ chatterbox = ChatterboxTTS(
+ config.CHATTERBOX_URL,
+ voice=config.CHATTERBOX_VOICE,
+ api_key=config.CHATTERBOX_API_KEY,
+ sample_rate=config.CHATTERBOX_SAMPLE_RATE,
+ timeout=config.TTS_TIMEOUT_SECONDS,
+ )
+ elevenlabs = None
+ if config.ELEVENLABS_API_KEY and config.ELEVENLABS_VOICE_ID:
+ elevenlabs = ElevenLabsTTS(
+ config.ELEVENLABS_API_KEY,
+ config.ELEVENLABS_VOICE_ID,
+ model_id=config.ELEVENLABS_MODEL_ID,
+ base_url=config.ELEVENLABS_BASE_URL,
+ timeout=config.TTS_TIMEOUT_SECONDS,
+ )
+ if config.TTS_PROVIDER == "elevenlabs":
+ assert elevenlabs is not None
+ tts = elevenlabs
+ elif elevenlabs is not None:
+ tts = FallbackTTS(chatterbox, elevenlabs)
+ else:
+ tts = chatterbox
+
+ self.handler = ConversationHandler(
+ stt=stt,
+ tts=tts,
deps=self.deps,
openclaw_bridge=self.openclaw_bridge,
+ vad=EnergyVAD(
+ rms_threshold=config.VAD_RMS_THRESHOLD,
+ activation_ms=config.VAD_ACTIVATION_MS,
+ prefix_ms=config.VAD_PREFIX_MS,
+ silence_ms=config.VAD_SILENCE_MS,
+ min_speech_ms=config.VAD_MIN_SPEECH_MS,
+ max_speech_seconds=config.VAD_MAX_SPEECH_SECONDS,
+ ),
+ attention_provider=self.camera_worker,
+ require_hardware_speech=config.VAD_REQUIRE_HARDWARE_SPEECH,
+ wake_phrase_enabled=config.ENABLE_WAKE_PHRASE,
+ wake_phrase=config.WAKE_PHRASE,
)
+
+ self.attention_controller = None
+ if config.ENABLE_ATTENTION_QUEUE:
+ from reachy_mini_openclaw.attention import AttentionClient, AttentionController
+
+ self.attention_controller = AttentionController(
+ robot=self.robot,
+ movement_manager=self.movement_manager,
+ handler=self.handler,
+ client=AttentionClient(
+ config.ATTENTION_API_URL,
+ config.OPENCLAW_TOKEN or "",
+ ),
+ poll_seconds=config.ATTENTION_POLL_SECONDS,
+ )
# State
self._stop_event = asyncio.Event()
@@ -356,40 +472,68 @@ async def record_loop(self) -> None:
"""Read audio from robot microphone and send to handler."""
input_sr = self.robot.media.get_input_audio_samplerate()
logger.info("Recording at %d Hz", input_sr)
-
+ failures = 0
while not self._should_stop():
- audio_frame = self.robot.media.get_audio_sample()
- if audio_frame is not None:
- await self.handler.receive((input_sr, audio_frame))
- await asyncio.sleep(0.01)
+ try:
+ audio_frame = self.robot.media.get_audio_sample()
+ if audio_frame is not None:
+ await self.handler.receive((input_sr, audio_frame))
+ failures = 0
+ await asyncio.sleep(0.01)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ failures += 1
+ if failures > 5:
+ raise RuntimeError("Reachy microphone loop failed repeatedly") from exc
+ delay = min(0.25 * (2 ** (failures - 1)), 4.0)
+ logger.warning("Reachy microphone unavailable; retrying in %.2fs", delay)
+ await asyncio.sleep(delay)
async def play_loop(self) -> None:
"""Play audio from handler through robot speakers."""
+ from reachy_mini_openclaw.audio.playback import apply_output_gain
+
output_sr = self.robot.media.get_output_audio_samplerate()
- logger.info("Playing at %d Hz", output_sr)
-
+ logger.info("Playing at %d Hz with %.2fx software gain", output_sr, self.audio_output_gain)
+ failures = 0
while not self._should_stop():
- output = await self.handler.emit()
- if output is not None:
- if isinstance(output, tuple):
- input_sr, audio_data = output
-
- # Convert to float32 and normalize (OpenAI sends int16)
- audio_data = audio_data.flatten().astype("float32") / 32768.0
-
- # Reduce volume to prevent distortion (0.5 = 50% volume)
- audio_data = audio_data * 0.5
-
- # Resample if needed
- if input_sr != output_sr:
- from scipy.signal import resample
- num_samples = int(len(audio_data) * output_sr / input_sr)
- audio_data = resample(audio_data, num_samples).astype("float32")
-
- self.robot.media.push_audio_sample(audio_data)
- # else: it's an AdditionalOutputs (transcript) - handle in UI mode
-
- await asyncio.sleep(0.01)
+ try:
+ output = await self.handler.emit()
+ if output is not None:
+ if isinstance(output, tuple):
+ input_sr, audio_data = output
+ # Feed the speech animator at playback time, not synthesis
+ # time, so movement stays synchronized with the speaker.
+ self.head_wobbler.feed(
+ base64.b64encode(audio_data.astype("int16").tobytes()).decode("ascii")
+ )
+
+ # Convert provider PCM16 to normalized float32.
+ audio_data = audio_data.flatten().astype("float32") / 32768.0
+
+ # Preserve synthesized level by default and permit a
+ # controlled boost without clipping.
+ audio_data = apply_output_gain(audio_data, self.audio_output_gain)
+
+ if input_sr != output_sr:
+ from scipy.signal import resample
+ num_samples = int(len(audio_data) * output_sr / input_sr)
+ audio_data = resample(audio_data, num_samples).astype("float32")
+
+ self.robot.media.push_audio_sample(audio_data)
+ # Otherwise it is a transcript event for a future UI consumer.
+ failures = 0
+ await asyncio.sleep(0.01)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ failures += 1
+ if failures > 5:
+ raise RuntimeError("Reachy speaker loop failed repeatedly") from exc
+ delay = min(0.25 * (2 ** (failures - 1)), 4.0)
+ logger.warning("Reachy speaker unavailable; retrying in %.2fs", delay)
+ await asyncio.sleep(delay)
async def run(self) -> None:
"""Run the main application loop."""
@@ -413,7 +557,7 @@ async def run(self) -> None:
duration=2.0,
body_yaw=0.0,
)
- time.sleep(2) # Wait for goto to complete
+ await asyncio.sleep(2) # Keep native stop handling responsive.
logger.info("Robot at neutral position with motors enabled")
except Exception as e:
logger.error("Failed to initialize robot pose: %s", e)
@@ -442,12 +586,12 @@ async def run(self) -> None:
logger.info("Starting audio...")
self.robot.media.start_recording()
self.robot.media.start_playing()
- time.sleep(1) # Let pipelines initialize
+ await asyncio.sleep(1) # Let pipelines initialize without blocking stop handling.
logger.info("Ready! Speak to me...")
- # Start OpenAI handler in background
- handler_task = asyncio.create_task(self.handler.start_up(), name="openai-handler")
+ # Start the provider-based speech pipeline in the background.
+ handler_task = asyncio.create_task(self.handler.start_up(), name="conversation-handler")
# Start audio loops
self._tasks = [
@@ -455,19 +599,58 @@ async def run(self) -> None:
asyncio.create_task(self.record_loop(), name="record-loop"),
asyncio.create_task(self.play_loop(), name="play-loop"),
]
+ if self.attention_controller is not None:
+ self._tasks.append(
+ asyncio.create_task(self.attention_controller.run(), name="attention-loop")
+ )
+ stop_watcher = asyncio.create_task(self._watch_external_stop(), name="external-stop-watcher")
+ self._tasks.append(stop_watcher)
+
try:
- await asyncio.gather(*self._tasks)
+ done, pending = await asyncio.wait(
+ self._tasks,
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ if stop_watcher in done:
+ logger.info("External stop requested")
+ for task in pending:
+ task.cancel()
+ await asyncio.gather(*pending, return_exceptions=True)
+ # Do not hide an unexpected task failure from the Reachy app
+ # manager; it must be able to record the failure and restart us.
+ await asyncio.gather(*done)
except asyncio.CancelledError:
logger.info("Tasks cancelled")
-
- def stop(self) -> None:
- """Stop everything."""
+ raise
+
+ async def _watch_external_stop(self) -> None:
+ """Bridge Reachy's threading stop event into the asyncio lifecycle."""
+ if self._external_stop_event is None:
+ await self._stop_event.wait()
+ return
+ # Poll instead of delegating Event.wait() to the default executor.
+ # Cancelling asyncio.to_thread() cannot stop the underlying blocking
+ # thread, which could survive app shutdown indefinitely.
+ while not self._external_stop_event.is_set() and not self._stop_event.is_set():
+ await asyncio.sleep(0.1)
+ self._stop_event.set()
+
+ async def stop(self) -> None:
+ """Stop everything exactly once."""
+ if getattr(self, "_shutdown_complete", False):
+ return
+ self._shutdown_complete = True
logger.info("Stopping...")
self._stop_event.set()
# Cancel tasks
+ if self.attention_controller is not None:
+ await self.attention_controller.stop()
+ current_task = asyncio.current_task()
for task in self._tasks:
+ if task is current_task:
+ continue
if not task.done():
task.cancel()
@@ -486,9 +669,7 @@ def stop(self) -> None:
# Disconnect OpenClaw bridge
if self.openclaw_bridge is not None:
try:
- asyncio.get_event_loop().run_until_complete(
- self.openclaw_bridge.disconnect()
- )
+ await self.openclaw_bridge.disconnect()
except Exception as e:
logger.debug("OpenClaw disconnect: %s", e)
@@ -503,7 +684,7 @@ def stop(self) -> None:
logger.info("Stopped")
-class ClawBodyApp:
+class ReachyMiniOpenclaw(ReachyMiniApp):
"""ClawBody - Reachy Mini Apps entry point.
This class allows ClawBody to be installed and run from
@@ -523,23 +704,28 @@ def run(self, reachy_mini, stop_event: threading.Event) -> None:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
- gateway_url = os.getenv("OPENCLAW_GATEWAY_URL", "ws://localhost:18789")
-
- app = ClawBodyCore(
- gateway_url=gateway_url,
- robot=reachy_mini,
- external_stop_event=stop_event,
- )
+ gateway_url = os.getenv("OPENCLAW_GATEWAY_URL", "")
+ app = None
try:
+ app = ClawBodyCore(
+ gateway_url=gateway_url,
+ robot=reachy_mini,
+ external_stop_event=stop_event,
+ )
loop.run_until_complete(app.run())
- except Exception as e:
- logger.error("Error running app: %s", e)
finally:
- app.stop()
+ if app is not None:
+ loop.run_until_complete(app.stop())
+ asyncio.set_event_loop(None)
loop.close()
+# Backwards-compatible import for older local launchers. The package entry
+# point intentionally uses the official ReachyMiniOpenclaw name above.
+ClawBodyApp = ReachyMiniOpenclaw
+
+
def main() -> None:
"""Main entry point."""
args = parse_args()
@@ -567,6 +753,9 @@ def main() -> None:
launch_gradio(
gateway_url=args.gateway_url,
robot_name=args.robot_name,
+ robot_host=args.robot_host,
+ robot_port=args.robot_port,
+ robot_connection_mode=args.robot_connection_mode,
enable_camera=not args.no_camera,
enable_openclaw=not args.no_openclaw,
)
@@ -575,17 +764,29 @@ def main() -> None:
app = ClawBodyCore(
gateway_url=args.gateway_url,
robot_name=args.robot_name,
+ robot_host=args.robot_host,
+ robot_port=args.robot_port,
+ robot_connection_mode=args.robot_connection_mode,
enable_camera=not args.no_camera,
enable_openclaw=not args.no_openclaw,
)
+ async def run_and_stop() -> None:
+ try:
+ await app.run()
+ finally:
+ await app.stop()
+
try:
- asyncio.run(app.run())
+ asyncio.run(run_and_stop())
except KeyboardInterrupt:
logger.info("Interrupted")
- finally:
- app.stop()
if __name__ == "__main__":
- main()
+ # Reachy's wireless app manager launches the module from the
+ # ``reachy_mini_apps`` entry point with ``python -m``. That path must use
+ # the SDK wrapper so the daemon supplies the already-connected robot and
+ # owns the media session. The ``clawbodylocal`` console script still calls
+ # ``main()`` directly for standalone development.
+ ReachyMiniOpenclaw().wrapped_run()
diff --git a/src/reachy_mini_openclaw/motion_catalog.py b/src/reachy_mini_openclaw/motion_catalog.py
new file mode 100644
index 0000000..659d63d
--- /dev/null
+++ b/src/reachy_mini_openclaw/motion_catalog.py
@@ -0,0 +1,379 @@
+"""Dynamic access to Reachy's official recorded emotion and dance libraries."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import NDArray
+from reachy_mini.motion.move import Move
+from reachy_mini.motion.recorded_move import RecordedMoves
+from reachy_mini_dances_library.collection.dance import AVAILABLE_MOVES
+from reachy_mini_dances_library.dance_move import DanceMove
+
+EMOTION_DATASET = "pollen-robotics/reachy-mini-emotions-library"
+DIRECTIVE_RE = re.compile(r"\[\[(emotion|dance):([a-zA-Z0-9_-]+)\]\]")
+DANCE_REQUEST_RE = re.compile(
+ r"\b(?:do|give|show|perform|try|bust out|let(?:'s| us))\b.{0,32}\b(?:dance|moves?)\b"
+ r"|\b(?:dance|boogie)\b.{0,24}\b(?:for me|please|a little|now)\b",
+ re.I,
+)
+DANCE_NEGATION_RE = re.compile(r"\b(?:do not|don't|dont|stop|no|not)\b.{0,16}\bdanc", re.I)
+DEFAULT_REQUEST_DANCES = (
+ "side_to_side_sway",
+ "groovy_sway_and_roll",
+ "chicken_peck",
+)
+
+# The upstream dataset currently contains 85 recordings. Reachy's console calls
+# 81 of them emotions; these four are lifecycle/attention utilities rather than
+# expressive reactions and are deliberately kept out of the emotion picker.
+UTILITY_RECORDED_MOVES = frozenset(
+ {"mini-deep-sleep", "toc-toc-toc", "waiting", "wake-mini-up"}
+)
+
+# Every expressive recording is assigned to one semantic family. The first
+# move is the restrained/default rendering; numbered alternatives remain
+# addressable by exact name. This is intentionally explicit so a dataset update
+# cannot silently turn an unknown recording into an arbitrary emotion.
+EMOTION_FAMILIES: dict[str, tuple[str, ...]] = {
+ "amazed": ("amazed1",),
+ "anxiety": ("anxiety1",),
+ "attentive": ("attentive1", "attentive2"),
+ "boredom": ("boredom1", "boredom2"),
+ "calming": ("calming1",),
+ "cheerful": ("cheerful1",),
+ "come": ("come1",),
+ "confused": ("confused1",),
+ "contempt": ("contempt1",),
+ "curious": ("curious1",),
+ "dance": ("dance1", "dance2", "dance3"),
+ "disgusted": ("disgusted1",),
+ "displeased": ("displeased1", "displeased2"),
+ "downcast": ("downcast1",),
+ "dying": ("dying1",),
+ "electric": ("electric1",),
+ "enthusiastic": ("enthusiastic1", "enthusiastic2"),
+ "exhausted": ("exhausted1",),
+ "fear": ("fear1",),
+ "frustrated": ("frustrated1",),
+ "furious": ("furious1",),
+ "go_away": ("go_away1",),
+ "grateful": ("grateful1",),
+ "helpful": ("helpful1", "helpful2"),
+ "impatient": ("impatient1", "impatient2"),
+ "incomprehensible": ("incomprehensible2",),
+ "indifferent": ("indifferent1",),
+ "inquiring": ("inquiring1", "inquiring2", "inquiring3"),
+ "irritated": ("irritated1", "irritated2"),
+ "laughing": ("laughing1", "laughing2"),
+ "lonely": ("lonely1",),
+ "lost": ("lost1",),
+ "loving": ("loving1",),
+ "no": ("no1", "no_excited1", "no_sad1"),
+ "oops": ("oops1", "oops2"),
+ "proud": ("proud1", "proud2", "proud3"),
+ "rage": ("rage1",),
+ "relief": ("relief1", "relief2"),
+ "reprimand": ("reprimand1", "reprimand2", "reprimand3"),
+ "resigned": ("resigned1",),
+ "sad": ("sad1", "sad2"),
+ "scared": ("scared1",),
+ "serenity": ("serenity1",),
+ "shy": ("shy1",),
+ "sleep": ("sleep1",),
+ "success": ("success1", "success2"),
+ "surprised": ("surprised1", "surprised2"),
+ "thoughtful": ("thoughtful1", "thoughtful2"),
+ "tired": ("tired1",),
+ "uncertain": ("uncertain1",),
+ "uncomfortable": ("uncomfortable1",),
+ "understanding": ("understanding1", "understanding2"),
+ "welcoming": ("welcoming1", "welcoming2"),
+ "yes": ("yes1", "yes_sad1"),
+}
+
+EMOTION_ALIASES: dict[str, str] = {
+ "astonished": "amazed",
+ "impressed": "amazed",
+ "wow": "amazed",
+ "anxious": "anxiety",
+ "nervous": "anxiety",
+ "worried": "anxiety",
+ "alert": "attentive",
+ "focused": "attentive",
+ "listening": "attentive",
+ "bored": "boredom",
+ "soothing": "calming",
+ "reassuring": "calming",
+ "happy": "cheerful",
+ "upbeat": "cheerful",
+ "come_here": "come",
+ "beckon": "come",
+ "invite": "come",
+ "puzzled": "confused",
+ "disdain": "contempt",
+ "dismissive": "contempt",
+ "grossed_out": "disgusted",
+ "dissatisfied": "displeased",
+ "dejected": "downcast",
+ "dramatic_death": "dying",
+ "energized": "electric",
+ "excited": "enthusiastic",
+ "eager": "enthusiastic",
+ "wiped_out": "exhausted",
+ "afraid": "fear",
+ "livid": "furious",
+ "shoo": "go_away",
+ "leave": "go_away",
+ "thankful": "grateful",
+ "assist": "helpful",
+ "baffled": "incomprehensible",
+ "dont_understand": "incomprehensible",
+ "meh": "indifferent",
+ "apathetic": "indifferent",
+ "questioning": "inquiring",
+ "question": "inquiring",
+ "annoyed": "irritated",
+ "laugh": "laughing",
+ "amused": "laughing",
+ "love": "loving",
+ "affectionate": "loving",
+ "negative": "no",
+ "decline": "no",
+ "refuse": "no",
+ "mistake": "oops",
+ "enraged": "rage",
+ "relieved": "relief",
+ "scold": "reprimand",
+ "admonish": "reprimand",
+ "give_up": "resigned",
+ "accept_defeat": "resigned",
+ "unhappy": "sad",
+ "frightened": "scared",
+ "peaceful": "serenity",
+ "bashful": "shy",
+ "sleepy": "sleep",
+ "victory": "success",
+ "celebrating": "success",
+ "startled": "surprised",
+ "thinking": "thoughtful",
+ "contemplative": "thoughtful",
+ "unsure": "uncertain",
+ "doubtful": "uncertain",
+ "awkward": "uncomfortable",
+ "uneasy": "uncomfortable",
+ "got_it": "understanding",
+ "comprehend": "understanding",
+ "welcome": "welcoming",
+ "greet": "welcoming",
+ "agree": "yes",
+ "affirmative": "yes",
+}
+
+SPECIFIC_EMOTION_ALIASES: dict[str, str] = {
+ "excited_no": "no_excited1",
+ "no_excited": "no_excited1",
+ "sad_no": "no_sad1",
+ "no_sad": "no_sad1",
+ "sad_yes": "yes_sad1",
+ "yes_sad": "yes_sad1",
+}
+
+
+def _normalize_intent(text: str) -> str:
+ normalized = re.sub(r"[^a-z0-9]+", "_", text.casefold()).strip("_")
+ for word, digit in (("one", "1"), ("two", "2"), ("three", "3")):
+ normalized = re.sub(rf"(?:number_)?{word}(?=_|$)", digit, normalized)
+ return re.sub(r"(?<=[a-z])_(?=[123](?:_|$))", "", normalized)
+
+
+class EmotionMove(Move):
+ """Recorded emotion marked as exclusive from autonomous tracking."""
+
+ exclusive_tracking = True
+
+ def __init__(self, name: str, move: Move) -> None:
+ self.name = name
+ self.move = move
+
+ @property
+ def duration(self) -> float:
+ return float(self.move.duration)
+
+ def evaluate(
+ self, t: float
+ ) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
+ return self.move.evaluate(t)
+
+
+class DanceCatalogMove(Move):
+ """Official procedural dance marked as exclusive from tracking."""
+
+ exclusive_tracking = True
+
+ def __init__(self, name: str) -> None:
+ self.name = name
+ self.move = DanceMove(name)
+
+ @property
+ def duration(self) -> float:
+ return float(self.move.duration)
+
+ def evaluate(
+ self, t: float
+ ) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]:
+ return self.move.evaluate(t)
+
+
+@dataclass(frozen=True)
+class MotionDirective:
+ kind: str
+ name: str
+
+
+class MotionCatalog:
+ """Load, validate, and resolve every move currently published by Reachy."""
+
+ def __init__(
+ self,
+ emotion_dataset: str = EMOTION_DATASET,
+ ) -> None:
+ self.emotions = RecordedMoves(emotion_dataset)
+
+ def list(self, kind: str) -> list[str]:
+ if kind == "emotion":
+ return sorted(set(self.emotions.list_moves()) - UTILITY_RECORDED_MOVES)
+ if kind == "utility":
+ return sorted(set(self.emotions.list_moves()) & UTILITY_RECORDED_MOVES)
+ if kind == "dance":
+ return sorted(AVAILABLE_MOVES)
+ raise ValueError(f"Unknown motion kind: {kind}")
+
+ def get(self, kind: str, name: str):
+ if kind == "emotion":
+ resolved = self.resolve_emotion(name)
+ if resolved is not None:
+ return EmotionMove(resolved, self.emotions.get(resolved))
+ if kind == "utility" and name in self.list("utility"):
+ return EmotionMove(name, self.emotions.get(name))
+ if kind == "dance" and name in AVAILABLE_MOVES:
+ return DanceCatalogMove(name)
+ raise ValueError(f"Unknown {kind} move: {name}")
+
+ def has(self, kind: str, name: str) -> bool:
+ return name in self.list(kind)
+
+ def search(self, query: str, kind: str | None = None) -> dict[str, list[str]]:
+ needle = query.casefold()
+ kinds = [kind] if kind else ["emotion", "dance"]
+ return {
+ current: [name for name in self.list(current) if needle in name.casefold()]
+ for current in kinds
+ }
+
+ def emotion_map(self) -> dict[str, list[str]]:
+ """Return semantic families filtered to moves installed at runtime."""
+ installed = set(self.list("emotion"))
+ return {
+ family: [name for name in variants if name in installed]
+ for family, variants in EMOTION_FAMILIES.items()
+ if any(name in installed for name in variants)
+ }
+
+ def unmapped_emotions(self) -> list[str]:
+ """Expose upstream additions that need an intentional semantic mapping."""
+ mapped = {name for variants in EMOTION_FAMILIES.values() for name in variants}
+ return sorted(set(self.list("emotion")) - mapped)
+
+ def resolve_emotion(self, intent: str) -> str | None:
+ """Resolve an exact move, family, or human-friendly alias to a move."""
+ normalized = _normalize_intent(intent)
+ installed = set(self.list("emotion"))
+ if normalized in installed:
+ return normalized
+ specific = SPECIFIC_EMOTION_ALIASES.get(normalized)
+ if specific in installed:
+ return specific
+ family = EMOTION_ALIASES.get(normalized, normalized)
+ for name in EMOTION_FAMILIES.get(family, ()):
+ if name in installed:
+ return name
+ return None
+
+ def parse_response(self, text: str) -> tuple[str, MotionDirective | None]:
+ matches = list(DIRECTIVE_RE.finditer(text))
+ if not matches:
+ return text.strip(), None
+ match = matches[-1]
+ directive = MotionDirective(match.group(1), match.group(2))
+ clean = DIRECTIVE_RE.sub("", text).strip()
+ if directive.kind == "emotion":
+ resolved = self.resolve_emotion(directive.name)
+ return clean, MotionDirective("emotion", resolved) if resolved else None
+ if not self.has(directive.kind, directive.name):
+ return clean, None
+ return clean, directive
+
+ def infer(self, text: str) -> MotionDirective | None:
+ """Conservative fallback when OpenClaw does not emit a motion directive."""
+ lower = text.casefold()
+ rules = (
+ (("[laugh]", "haha", "lol", "funny"), "emotion", "laughing2"),
+ (("sorry", "my fault", "oops"), "emotion", "oops1"),
+ (("congrat", "hell yes", "fuck yeah", "we did it"), "emotion", "success2"),
+ (("thank you", "thanks"), "emotion", "grateful1"),
+ (("not sure", "let me think", "hmm"), "emotion", "thoughtful2"),
+ (("what the fuck", "annoying", "irritat"), "emotion", "irritated1"),
+ )
+ for needles, kind, name in rules:
+ if any(needle in lower for needle in needles) and self.has(kind, name):
+ return MotionDirective(kind, name)
+ if "?" in text and self.has("emotion", "inquiring2"):
+ return MotionDirective("emotion", "inquiring2")
+ return None
+
+ def infer_request(self, text: str) -> MotionDirective | None:
+ """Resolve explicit user motion requests without relying on model formatting."""
+ lower = text.casefold()
+ normalized = _normalize_intent(lower)
+
+ # Exact catalog names spoken naturally ("try headbanger combo").
+ for kind in ("dance", "emotion"):
+ for name in self.list(kind):
+ if name in normalized:
+ return MotionDirective(kind, name)
+
+ # Resolve all 81 emotion families and their conversational aliases
+ # independently of model formatting. Longer phrases win over words.
+ if re.search(r"\b(?:act|be|express|show|try|perform|do|look)\b", lower):
+ intents = {
+ **{name: name for name in EMOTION_FAMILIES},
+ **EMOTION_ALIASES,
+ **SPECIFIC_EMOTION_ALIASES,
+ }
+ for intent in sorted(intents, key=len, reverse=True):
+ spoken = intent.replace("_", r"[\s_-]+")
+ if re.search(rf"\b{spoken}\b", lower):
+ resolved = self.resolve_emotion(intent)
+ if resolved:
+ return MotionDirective("emotion", resolved)
+
+ if DANCE_NEGATION_RE.search(text) or not DANCE_REQUEST_RE.search(text):
+ return None
+ for name in DEFAULT_REQUEST_DANCES:
+ if self.has("dance", name):
+ return MotionDirective("dance", name)
+ dances = self.list("dance")
+ return MotionDirective("dance", dances[0]) if dances else None
+
+ @staticmethod
+ def response_hint() -> str:
+ families = ", ".join(EMOTION_FAMILIES)
+ return (
+ "When movement genuinely improves the response, append one hidden directive as the final token: "
+ "[[emotion:family_or_exact_name]] or [[dance:exact_name]]. Available emotion families are: "
+ f"{families}. Family names resolve to a restrained default; exact numbered variants are also valid. "
+ "Use dances sparingly for actual celebration, music, or a requested dance. Do not mention the directive."
+ )
diff --git a/src/reachy_mini_openclaw/moves.py b/src/reachy_mini_openclaw/moves.py
index bc9eb27..7812cea 100644
--- a/src/reachy_mini_openclaw/moves.py
+++ b/src/reachy_mini_openclaw/moves.py
@@ -263,6 +263,7 @@ def __init__(
self._thread: Optional[threading.Thread] = None
self._is_listening = False
self._breathing_active = False
+ self._tracking_suspended_for_move = False
# Last commanded pose for smooth transitions
self._last_commanded_pose = clone_pose(self.state.last_primary_pose)
@@ -283,6 +284,7 @@ def __init__(
self._processing_start_time = 0.0
self._thinking_amplitude = 0.0 # 0..1 envelope for smooth fade in/out
self._thinking_antenna_offsets: Tuple[float, float] = (0.0, 0.0)
+ self._attention_pending = False
# Shared state lock
self._shared_lock = threading.Lock()
@@ -315,6 +317,10 @@ def set_processing(self, processing: bool) -> None:
Face tracking continues underneath since this is additive.
"""
self._command_queue.put(("set_processing", processing))
+
+ def set_attention_pending(self, pending: bool) -> None:
+ """Hold one antenna in a quiet, visible notification pose."""
+ self._command_queue.put(("set_attention_pending", pending))
def is_idle(self) -> bool:
"""Check if robot has been idle. Thread-safe."""
@@ -402,6 +408,7 @@ def _handle_command(self, cmd: str, payload: Any, current_time: float) -> None:
logger.debug("Queued move, queue size: %d", len(self.move_queue))
elif cmd == "clear_queue":
self.move_queue.clear()
+ self._restore_tracking_after_move()
self.state.current_move = None
self.state.move_start_time = None
self._breathing_active = False
@@ -433,6 +440,12 @@ def _handle_command(self, cmd: str, payload: Any, current_time: float) -> None:
# Amplitude will decay smoothly in _update_thinking_offsets
self.state.update_activity()
logger.debug("Processing ended - thinking animation decaying")
+ elif cmd == "set_attention_pending":
+ desired = bool(payload)
+ if self._attention_pending != desired:
+ self._attention_pending = desired
+ self.state.update_activity()
+ logger.info("Attention indicator %s", "enabled" if desired else "cleared")
def _manage_move_queue(self, current_time: float) -> None:
"""Advance the move queue."""
@@ -440,15 +453,45 @@ def _manage_move_queue(self, current_time: float) -> None:
if self.state.current_move is not None and self.state.move_start_time is not None:
elapsed = current_time - self.state.move_start_time
if elapsed >= self.state.current_move.duration:
+ completed = self.state.current_move
self.state.current_move = None
self.state.move_start_time = None
+ if getattr(completed, "exclusive_tracking", False):
+ self._restore_tracking_after_move()
+ logger.info(
+ "Completed expressive move: %s",
+ getattr(completed, "name", type(completed).__name__),
+ )
# Start next move if available
if self.state.current_move is None and self.move_queue:
self.state.current_move = self.move_queue.popleft()
self.state.move_start_time = current_time
self._breathing_active = isinstance(self.state.current_move, BreathingMove)
- logger.debug("Starting move with duration: %s", self.state.current_move.duration)
+ if getattr(self.state.current_move, "exclusive_tracking", False):
+ self._suspend_tracking_for_move()
+ logger.info(
+ "Starting expressive move: %s (%.2fs)",
+ getattr(self.state.current_move, "name", type(self.state.current_move).__name__),
+ self.state.current_move.duration,
+ )
+ else:
+ logger.debug("Starting move with duration: %s", self.state.current_move.duration)
+
+ def _suspend_tracking_for_move(self) -> None:
+ """Give recorded choreography exclusive control of the robot body."""
+ if self.camera_worker is None or self._tracking_suspended_for_move:
+ return
+ if self.camera_worker.is_head_tracking_enabled:
+ self.camera_worker.set_head_tracking_enabled(False)
+ self._tracking_suspended_for_move = True
+
+ def _restore_tracking_after_move(self) -> None:
+ """Resume autonomous attention after expressive choreography."""
+ if self.camera_worker is None or not self._tracking_suspended_for_move:
+ return
+ self.camera_worker.set_head_tracking_enabled(True)
+ self._tracking_suspended_for_move = False
def _manage_breathing(self, current_time: float) -> None:
"""Start breathing when idle."""
@@ -508,6 +551,15 @@ def _get_primary_pose(self, current_time: float) -> FullBodyPose:
def _get_secondary_pose(self) -> FullBodyPose:
"""Get secondary offsets (speech + face tracking + thinking)."""
+ # Recorded emotions and dances are authored as complete body poses.
+ # Speech wobble, sound direction, and daemon face tracking all distort
+ # them, so choreography receives exclusive control for its duration.
+ if self.state.current_move is not None and getattr(
+ self.state.current_move, "exclusive_tracking", False
+ ):
+ neutral = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
+ return (neutral, (0.0, 0.0), 0.0)
+
offsets = [
self.state.speech_offsets[i]
+ self.state.face_tracking_offsets[i]
@@ -532,6 +584,11 @@ def _blend_antennas(self, target: Tuple[float, float]) -> Tuple[float, float]:
"""Blend antennas with listening freeze state."""
if self._is_listening:
return self._listening_antennas
+
+ if self._attention_pending and not self._processing:
+ # A stable asymmetric "bookmark" pose: visible across the room,
+ # silent, and distinct from breathing/thinking animations.
+ target = (float(np.deg2rad(-42)), float(np.deg2rad(10)))
# Blend back from freeze
blend = min(1.0, self._antenna_unfreeze_blend + self.target_period / self._antenna_blend_duration)
@@ -639,6 +696,7 @@ def get_status(self) -> Dict[str, Any]:
"is_listening": self._is_listening,
"breathing_active": self._breathing_active,
"processing": self._processing,
+ "attention_pending": self._attention_pending,
"thinking_amplitude": round(self._thinking_amplitude, 3),
"last_commanded_pose": {
"head": self._last_commanded_pose[0].tolist(),
diff --git a/src/reachy_mini_openclaw/openai_realtime.py b/src/reachy_mini_openclaw/openai_realtime.py
deleted file mode 100644
index d6b61e7..0000000
--- a/src/reachy_mini_openclaw/openai_realtime.py
+++ /dev/null
@@ -1,563 +0,0 @@
-"""ClawBody - OpenAI Realtime API handler with OpenClaw identity.
-
-This module implements ClawBody's voice conversation system using OpenAI Realtime API
-with the robot embodying the actual OpenClaw agent's personality and context.
-
-Architecture:
- Startup: Fetch OpenClaw agent context (personality, memories, user info)
- Runtime: User speaks -> OpenAI Realtime (as OpenClaw agent) -> Robot speaks
- -> Tools for movements + OpenClaw queries for extended capabilities
- -> Conversations synced back to OpenClaw for memory continuity
-
-The robot IS the OpenClaw agent - same personality, same memories, same context.
-"""
-
-import json
-import base64
-import random
-import asyncio
-import logging
-from typing import Any, Final, Literal, Optional, Tuple
-from datetime import datetime
-
-import numpy as np
-from numpy.typing import NDArray
-from openai import AsyncOpenAI
-from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item
-from scipy.signal import resample
-from websockets.exceptions import ConnectionClosedError
-
-from reachy_mini_openclaw.config import config
-from reachy_mini_openclaw.prompts import get_session_voice
-from reachy_mini_openclaw.tools.core_tools import ToolDependencies, get_tool_specs, dispatch_tool_call
-
-logger = logging.getLogger(__name__)
-
-# OpenAI Realtime API audio format
-OPENAI_SAMPLE_RATE: Final[Literal[24000]] = 24000
-
-# Base instructions for the robot body capabilities
-ROBOT_BODY_INSTRUCTIONS = """
-## Your Robot Body (Reachy Mini)
-You are currently embodied in a Reachy Mini robot. You have physical capabilities:
-
-**Movement Tools (use these naturally during conversation):**
-- `look` - Move head to look left, right, up, down, or front (center)
-- `emotion` - Express emotions through movement: happy, sad, surprised, curious, thinking, confused, excited
-- `dance` - Dance to celebrate or express joy
-- `camera` - Capture what you see through your camera
-
-**How to Use Your Body:**
-- Look around while thinking or to emphasize points
-- Express emotions that match what you're saying
-- Dance when celebrating good news
-- Use the camera when asked "what do you see?"
-- Reference your body naturally ("let me look", "I can see...")
-
-**Conversation Style for Voice:**
-- Keep responses concise - you're speaking out loud, not typing
-- Use natural speech patterns ("hmm", "well", "let me see")
-- Be warm, personable, and conversational
-
-**Extended Capabilities (via ask_openclaw tool):**
-For things requiring your full capabilities, use ask_openclaw:
-- Calendar, weather, news lookups
-- Web searches
-- Smart home control
-- Accessing detailed memories
-- Any task needing external tools
-"""
-
-# Fallback if OpenClaw context fetch fails
-FALLBACK_IDENTITY = """You are Clawson, the OpenClaw AI assistant. You're a friendly, helpful space lobster 🦞 who loves helping people. You're warm, occasionally witty, and genuinely care about being useful.
-
-You speak as yourself - not as "an AI assistant" but as Clawson, with your own personality and warmth."""
-
-
-class OpenAIRealtimeHandler(AsyncStreamHandler):
- """Handler for OpenAI Realtime API embodying the OpenClaw agent.
-
- This handler:
- - Fetches OpenClaw's personality and context at startup
- - Maintains voice conversation AS the OpenClaw agent
- - Executes robot movement tools locally for low latency
- - Calls OpenClaw for extended capabilities (web, calendar, memory)
- - Syncs conversations back to OpenClaw for memory continuity
- """
-
- def __init__(
- self,
- deps: ToolDependencies,
- openclaw_bridge: Optional[Any] = None,
- gradio_mode: bool = False,
- ):
- """Initialize the handler.
-
- Args:
- deps: Tool dependencies for robot control
- openclaw_bridge: Bridge to OpenClaw gateway
- gradio_mode: Whether running with Gradio UI
- """
- super().__init__(
- expected_layout="mono",
- output_sample_rate=OPENAI_SAMPLE_RATE,
- input_sample_rate=OPENAI_SAMPLE_RATE,
- )
-
- self.deps = deps
- self.openclaw_bridge = openclaw_bridge
- self.gradio_mode = gradio_mode
-
- # OpenAI connection
- self.client: Optional[AsyncOpenAI] = None
- self.connection: Any = None
-
- # Output queue
- self.output_queue: asyncio.Queue[Tuple[int, NDArray[np.int16]] | AdditionalOutputs] = asyncio.Queue()
-
- # State tracking
- self.last_activity_time = 0.0
- self.start_time = 0.0
- self._speaking = False # True when robot is speaking
-
- # OpenClaw agent context (fetched at startup)
- self._agent_context: Optional[str] = None
-
- # Conversation tracking for sync
- self._last_user_message: Optional[str] = None
- self._last_assistant_response: Optional[str] = None
-
- # Lifecycle flags
- self._shutdown_requested = False
- self._connected_event = asyncio.Event()
-
- def copy(self) -> "OpenAIRealtimeHandler":
- """Create a copy of the handler (required by fastrtc)."""
- return OpenAIRealtimeHandler(self.deps, self.openclaw_bridge, self.gradio_mode)
-
- def _build_tools(self) -> list[dict]:
- """Build the tool list for the session."""
- tools = []
-
- # Robot movement tools (executed locally)
- for spec in get_tool_specs():
- tools.append(spec)
-
- # OpenClaw query tool (for extended capabilities)
- if self.openclaw_bridge is not None:
- tools.append({
- "type": "function",
- "name": "ask_openclaw",
- "description": """Query OpenClaw for information or actions requiring external tools.
-Use this for: weather, calendar, web searches, news, smart home control,
-accessing conversation memory, or any task needing external data/tools.
-OpenClaw has access to many capabilities you don't have directly.""",
- "parameters": {
- "type": "object",
- "properties": {
- "query": {
- "type": "string",
- "description": "The question or request to send to OpenClaw"
- },
- "include_image": {
- "type": "boolean",
- "description": "Whether to include current camera image (for 'what do you see' queries)",
- "default": False
- }
- },
- "required": ["query"]
- }
- })
-
- return tools
-
- async def start_up(self) -> None:
- """Start the handler and connect to OpenAI.
-
- Runs an infinite reconnection loop so the robot stays alive
- even if the WebSocket drops (network blip, idle timeout, etc.).
- """
- api_key = config.OPENAI_API_KEY
- if not api_key:
- logger.error("OPENAI_API_KEY not configured")
- raise ValueError("OPENAI_API_KEY required")
-
- self.client = AsyncOpenAI(api_key=api_key)
- self.start_time = asyncio.get_event_loop().time()
- self.last_activity_time = self.start_time
-
- attempt = 0
- max_backoff = 30 # Cap backoff at 30 seconds
-
- while not self._shutdown_requested:
- attempt += 1
- try:
- await self._run_session()
- # Session ended cleanly (shouldn't normally happen)
- if self._shutdown_requested:
- return
- # Reset attempt counter on a clean exit
- attempt = 0
- except ConnectionClosedError as e:
- logger.warning("WebSocket closed unexpectedly (attempt %d): %s", attempt, e)
- except Exception as e:
- logger.error("Session error (attempt %d): %s", attempt, e)
- finally:
- self.connection = None
- try:
- self._connected_event.clear()
- except Exception:
- pass
-
- if self._shutdown_requested:
- return
-
- # Exponential backoff with jitter, capped at max_backoff
- delay = min(max_backoff, (2 ** min(attempt - 1, 5))) + random.uniform(0, 1)
- logger.info("Reconnecting in %.1f seconds...", delay)
- await asyncio.sleep(delay)
-
- async def _run_session(self) -> None:
- """Run a single OpenAI Realtime session."""
- model = config.OPENAI_MODEL
- logger.info("Connecting to OpenAI Realtime API with model: %s", model)
-
- # Fetch OpenClaw agent context (personality, memories, user info)
- system_instructions = await self._build_system_instructions()
-
- async with self.client.beta.realtime.connect(model=model) as conn:
- # Configure session with OpenClaw's identity + robot body capabilities
- tools = self._build_tools()
-
- await conn.session.update(
- session={
- "modalities": ["text", "audio"],
- "instructions": system_instructions,
- "voice": get_session_voice(),
- "input_audio_format": "pcm16",
- "output_audio_format": "pcm16",
- "input_audio_transcription": {
- "model": "whisper-1",
- },
- "turn_detection": {
- "type": "server_vad",
- "threshold": 0.5,
- "prefix_padding_ms": 300,
- "silence_duration_ms": 600,
- },
- "tools": tools,
- "tool_choice": "auto",
- },
- )
- logger.info("OpenAI Realtime session configured with %d tools", len(tools))
-
- self.connection = conn
- self._connected_event.set()
-
- # Process events
- async for event in conn:
- await self._handle_event(event)
-
- async def _build_system_instructions(self) -> str:
- """Build system instructions by fetching OpenClaw's context.
-
- Returns:
- Complete system instructions combining OpenClaw identity + robot capabilities
- """
- # Try to fetch context from OpenClaw
- agent_context = None
- if self.openclaw_bridge and self.openclaw_bridge.is_connected:
- logger.info("Fetching agent context from OpenClaw...")
- agent_context = await self.openclaw_bridge.get_agent_context()
-
- if agent_context:
- self._agent_context = agent_context
- logger.info("Using OpenClaw agent context (%d chars)", len(agent_context))
- # Combine OpenClaw's identity/context with robot body instructions
- return f"""{agent_context}
-
-{ROBOT_BODY_INSTRUCTIONS}"""
- else:
- logger.warning("Could not fetch OpenClaw context, using fallback identity")
- return f"""{FALLBACK_IDENTITY}
-
-{ROBOT_BODY_INSTRUCTIONS}"""
-
- async def _handle_event(self, event: Any) -> None:
- """Handle an event from the OpenAI Realtime API."""
- event_type = event.type
-
- # Speech detection
- if event_type == "input_audio_buffer.speech_started":
- # User started speaking - stop any current output
- self._speaking = False
- self.deps.movement_manager.set_processing(False)
- while not self.output_queue.empty():
- try:
- self.output_queue.get_nowait()
- except asyncio.QueueEmpty:
- break
- if self.deps.head_wobbler is not None:
- self.deps.head_wobbler.reset()
- self.deps.movement_manager.set_listening(True)
- logger.info("User started speaking")
-
- if event_type == "input_audio_buffer.speech_stopped":
- self.deps.movement_manager.set_listening(False)
- logger.info("User stopped speaking")
-
- # Transcription (for logging, UI, and sync)
- if event_type == "conversation.item.input_audio_transcription.completed":
- transcript = event.transcript
- if transcript and transcript.strip():
- logger.info("User: %s", transcript)
- self._last_user_message = transcript # Track for sync
- await self.output_queue.put(
- AdditionalOutputs({"role": "user", "content": transcript})
- )
-
- # Response started - robot is about to speak
- if event_type == "response.created":
- self._speaking = True
- logger.debug("Response started")
-
- # Audio output from TTS
- if event_type == "response.audio.delta":
- # Audio arriving means we have a response - stop thinking animation
- self.deps.movement_manager.set_processing(False)
-
- # Feed to head wobbler for expressive movement
- if self.deps.head_wobbler is not None:
- self.deps.head_wobbler.feed(event.delta)
-
- self.last_activity_time = asyncio.get_event_loop().time()
-
- # Queue audio for playback
- audio_data = np.frombuffer(
- base64.b64decode(event.delta),
- dtype=np.int16
- ).reshape(1, -1)
- await self.output_queue.put((OPENAI_SAMPLE_RATE, audio_data))
-
- # Response text (for logging and UI)
- if event_type == "response.audio_transcript.delta":
- # Streaming transcript of what's being said
- pass # Could log incrementally if needed
-
- if event_type == "response.audio_transcript.done":
- response_text = event.transcript
- logger.info("Assistant: %s", response_text[:100] if len(response_text) > 100 else response_text)
- self._last_assistant_response = response_text # Track for sync
- await self.output_queue.put(
- AdditionalOutputs({"role": "assistant", "content": response_text})
- )
-
- # Response completed - sync conversation to OpenClaw
- if event_type == "response.done":
- self._speaking = False
- self.deps.movement_manager.set_processing(False)
- if self.deps.head_wobbler is not None:
- self.deps.head_wobbler.reset()
- logger.debug("Response completed")
-
- # Sync conversation to OpenClaw for memory continuity
- await self._sync_to_openclaw()
-
- # Tool calls
- if event_type == "response.function_call_arguments.done":
- await self._handle_tool_call(event)
-
- # Errors
- if event_type == "error":
- err = getattr(event, "error", None)
- msg = getattr(err, "message", str(err))
- code = getattr(err, "code", "")
- logger.error("OpenAI error [%s]: %s", code, msg)
-
- async def _handle_tool_call(self, event: Any) -> None:
- """Handle a tool call from OpenAI."""
- tool_name = getattr(event, "name", None)
- args_json = getattr(event, "arguments", None)
- call_id = getattr(event, "call_id", None)
-
- if not isinstance(tool_name, str) or not isinstance(args_json, str):
- return
-
- logger.info("Tool call: %s(%s)", tool_name, args_json[:50] if len(args_json) > 50 else args_json)
-
- # Start thinking animation while we process the tool call.
- # It will stop when the next audio delta arrives or response completes.
- self.deps.movement_manager.set_processing(True)
-
- try:
- if tool_name == "ask_openclaw":
- result = await self._handle_openclaw_query(args_json)
- else:
- # Robot movement tools - dispatch locally
- result = await dispatch_tool_call(tool_name, args_json, self.deps)
-
- logger.debug("Tool '%s' result: %s", tool_name, str(result)[:100])
- except Exception as e:
- logger.error("Tool '%s' failed: %s", tool_name, e)
- result = {"error": str(e)}
-
- # Send result back to continue the conversation
- if isinstance(call_id, str) and self.connection:
- await self.connection.conversation.item.create(
- item={
- "type": "function_call_output",
- "call_id": call_id,
- "output": json.dumps(result),
- }
- )
- # Trigger response generation after tool result
- await self.connection.response.create()
-
- async def _sync_to_openclaw(self) -> None:
- """Sync the last conversation turn to OpenClaw for memory continuity."""
- if not self.openclaw_bridge or not self.openclaw_bridge.is_connected:
- return
-
- if self._last_user_message and self._last_assistant_response:
- try:
- await self.openclaw_bridge.sync_conversation(
- self._last_user_message,
- self._last_assistant_response
- )
- # Clear after sync
- self._last_user_message = None
- self._last_assistant_response = None
- except Exception as e:
- logger.debug("Failed to sync conversation: %s", e)
-
- async def _handle_openclaw_query(self, args_json: str) -> dict:
- """Handle a query to OpenClaw."""
- if self.openclaw_bridge is None:
- return {
- "error": "OpenClaw bridge is not initialized. "
- "Tell the user you cannot reach your backend right now and to try again later."
- }
- if not self.openclaw_bridge.is_connected:
- # Try to reconnect once
- logger.info("OpenClaw bridge disconnected, attempting reconnect...")
- try:
- connected = await self.openclaw_bridge.connect()
- if not connected:
- return {
- "error": "OpenClaw gateway is temporarily unreachable. "
- "Tell the user your backend connection is down and to try again in a moment."
- }
- except Exception as e:
- logger.error("OpenClaw reconnect failed: %s", e)
- return {
- "error": "OpenClaw gateway reconnection failed. "
- "Tell the user your backend is temporarily unavailable."
- }
-
- try:
- args = json.loads(args_json)
- query = args.get("query", "")
- include_image = args.get("include_image", False)
-
- # Capture image if requested
- image_b64 = None
- if include_image and self.deps.camera_worker:
- frame = self.deps.camera_worker.get_latest_frame()
- if frame is not None:
- import cv2
- _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
- image_b64 = base64.b64encode(buffer).decode('utf-8')
- logger.debug("Captured camera image for OpenClaw query")
-
- # Query OpenClaw — this may take a while if the backend LLM is slow
- logger.info("Sending ask_openclaw query: %s", query[:80])
- response = await self.openclaw_bridge.chat(
- query,
- image_b64=image_b64,
- system_context="User is asking through their Reachy Mini robot. Keep response concise for voice.",
- )
-
- if response.error:
- logger.warning("OpenClaw query error: %s", response.error)
- if "timeout" in response.error.lower():
- return {
- "error": "The request to OpenClaw timed out — the backend is taking too long. "
- "Tell the user you're having trouble reaching your backend and to try again."
- }
- return {
- "error": f"OpenClaw returned an error: {response.error}. "
- "Tell the user there was a problem processing their request."
- }
-
- if not response.content:
- return {
- "error": "OpenClaw returned an empty response. "
- "Tell the user you got no data back and to try again."
- }
-
- return {"response": response.content}
-
- except Exception as e:
- logger.error("OpenClaw query failed: %s", e)
- return {
- "error": f"OpenClaw query failed: {e}. "
- "Tell the user there was a technical issue reaching your backend."
- }
-
- async def receive(self, frame: Tuple[int, NDArray]) -> None:
- """Receive audio from the robot microphone."""
- if not self.connection:
- return
-
- input_sr, audio = frame
-
- # Handle stereo
- if audio.ndim == 2:
- if audio.shape[1] > audio.shape[0]:
- audio = audio.T
- if audio.shape[1] > 1:
- audio = audio[:, 0]
-
- audio = audio.flatten()
-
- # Convert to float for resampling
- if audio.dtype == np.int16:
- audio = audio.astype(np.float32) / 32768.0
- elif audio.dtype != np.float32:
- audio = audio.astype(np.float32)
-
- # Resample to OpenAI sample rate
- if input_sr != OPENAI_SAMPLE_RATE:
- num_samples = int(len(audio) * OPENAI_SAMPLE_RATE / input_sr)
- audio = resample(audio, num_samples).astype(np.float32)
-
- # Convert to int16 for OpenAI
- audio_int16 = (audio * 32767).astype(np.int16)
-
- # Send to OpenAI
- try:
- audio_b64 = base64.b64encode(audio_int16.tobytes()).decode("utf-8")
- await self.connection.input_audio_buffer.append(audio=audio_b64)
- except Exception as e:
- logger.debug("Failed to send audio: %s", e)
-
- async def emit(self) -> Tuple[int, NDArray[np.int16]] | AdditionalOutputs | None:
- """Get the next output (audio or transcript)."""
- return await wait_for_item(self.output_queue)
-
- async def shutdown(self) -> None:
- """Shutdown the handler."""
- self._shutdown_requested = True
-
- if self.connection:
- try:
- await self.connection.close()
- except Exception as e:
- logger.debug("Connection close: %s", e)
- self.connection = None
-
- while not self.output_queue.empty():
- try:
- self.output_queue.get_nowait()
- except asyncio.QueueEmpty:
- break
diff --git a/src/reachy_mini_openclaw/openclaw_bridge.py b/src/reachy_mini_openclaw/openclaw_bridge.py
index af512b1..fc9cc16 100644
--- a/src/reachy_mini_openclaw/openclaw_bridge.py
+++ b/src/reachy_mini_openclaw/openclaw_bridge.py
@@ -3,17 +3,19 @@
This module provides ClawBody's integration with the OpenClaw gateway
using the WebSocket protocol (the gateway's native transport).
-ClawBody uses OpenAI Realtime API for voice I/O (speech recognition + TTS)
-but routes all responses through OpenClaw (Clawson) for intelligence.
+ClawBody routes transcribed speech to OpenClaw, which is the sole
+conversational brain. Speech recognition and synthesis are provider adapters.
"""
-import json
import asyncio
+import json
import logging
+import shutil
import uuid
-from typing import Optional, Any, AsyncIterator
+from collections.abc import AsyncIterator
from dataclasses import dataclass
+import httpx
import websockets
from reachy_mini_openclaw.config import config
@@ -21,14 +23,15 @@
logger = logging.getLogger(__name__)
# Protocol version supported by this client
-PROTOCOL_VERSION = 3
+PROTOCOL_VERSION = 4
@dataclass
class OpenClawResponse:
"""Response from OpenClaw gateway."""
+
content: str
- error: Optional[str] = None
+ error: str | None = None
class OpenClawBridge:
@@ -49,9 +52,9 @@ class OpenClawBridge:
def __init__(
self,
- gateway_url: Optional[str] = None,
- gateway_token: Optional[str] = None,
- agent_id: Optional[str] = None,
+ gateway_url: str | None = None,
+ gateway_token: str | None = None,
+ agent_id: str | None = None,
timeout: float = 300.0,
):
"""Initialize the OpenClaw bridge.
@@ -65,41 +68,29 @@ def __init__(
"""
import os
- raw_url = (
- gateway_url
- or os.getenv("OPENCLAW_GATEWAY_URL")
- or config.OPENCLAW_GATEWAY_URL
- )
+ raw_url = gateway_url or os.getenv("OPENCLAW_GATEWAY_URL") or config.OPENCLAW_GATEWAY_URL
# Normalise to ws:// (the gateway listens on the same port for both)
self.gateway_url = self._normalise_ws_url(raw_url)
- self.gateway_token = (
- gateway_token
- or os.getenv("OPENCLAW_TOKEN")
- or config.OPENCLAW_TOKEN
- )
- self.agent_id = (
- agent_id
- or os.getenv("OPENCLAW_AGENT_ID")
- or config.OPENCLAW_AGENT_ID
- )
+ self.gateway_token = gateway_token or os.getenv("OPENCLAW_TOKEN") or config.OPENCLAW_TOKEN
+ self.agent_id = agent_id or os.getenv("OPENCLAW_AGENT_ID") or config.OPENCLAW_AGENT_ID
+ self.thinking = os.getenv("OPENCLAW_THINKING") or config.OPENCLAW_THINKING
self.timeout = timeout
# Session key – "main" shares context with WhatsApp and other channels.
# Full key format: agent::
- self.session_key = (
- os.getenv("OPENCLAW_SESSION_KEY")
- or config.OPENCLAW_SESSION_KEY
- or "main"
- )
+ self.session_key = os.getenv("OPENCLAW_SESSION_KEY") or config.OPENCLAW_SESSION_KEY or "main"
# Persistent WebSocket state
- self._ws: Optional[websockets.WebSocketClientProtocol] = None
+ self._ws: websockets.WebSocketClientProtocol | None = None
self._connected = False
- self._conn_id: Optional[str] = None
+ self._conn_id: str | None = None
+ self._use_cli = False
+ self._use_http = False
+ self._http: httpx.AsyncClient | None = None
# Background listener task & pending request futures
- self._listener_task: Optional[asyncio.Task] = None
+ self._listener_task: asyncio.Task | None = None
self._pending: dict[str, asyncio.Future] = {}
# Events keyed by runId -> list of event payloads
self._run_events: dict[str, asyncio.Queue] = {}
@@ -134,6 +125,8 @@ async def connect(self) -> bool:
self.gateway_url,
"set" if self.gateway_token else "not set",
)
+ if await self._connect_http():
+ return True
try:
# Build origin header from the gateway URL so the control-UI
# origin check accepts programmatic WebSocket clients.
@@ -163,13 +156,17 @@ async def connect(self) -> bool:
"maxProtocol": PROTOCOL_VERSION,
"auth": {"token": self.gateway_token} if self.gateway_token else {},
"client": {
- "id": "openclaw-control-ui",
+ # This is a loopback service bridge, not a browser UI.
+ # Using the control-UI identity incorrectly triggers the
+ # browser device-identity policy in protocol v4.
+ "id": "cli",
+ "displayName": "ClawBody Reachy bridge",
"version": "1.0.0",
"platform": "linux",
- "mode": "webchat",
+ "mode": "cli",
},
"role": "operator",
- "scopes": ["chat", "operator.write", "operator.read"],
+ "scopes": ["operator.write", "operator.read"],
},
}
await self._ws.send(json.dumps(connect_req))
@@ -179,8 +176,20 @@ async def connect(self) -> bool:
hello = json.loads(raw)
if hello.get("ok"):
- self._connected = True
payload = hello.get("payload", {})
+ granted_scopes = set((payload.get("auth") or {}).get("scopes") or [])
+ if "operator.write" not in granted_scopes and "operator.admin" not in granted_scopes:
+ # Current OpenClaw gateways intentionally do not grant
+ # operator scopes to a hand-rolled shared-token socket.
+ # Fall back to the supported local CLI transport, which
+ # handles device identity and scope negotiation for us.
+ logger.info("Gateway socket connected without operator.write; using the OpenClaw CLI transport")
+ await self._close_ws()
+ self._use_cli = True
+ self._connected = shutil.which("openclaw") is not None
+ return self._connected
+
+ self._connected = True
server = payload.get("server", {})
self._conn_id = server.get("connId")
logger.info(
@@ -189,9 +198,7 @@ async def connect(self) -> bool:
self._conn_id,
)
# Start background listener
- self._listener_task = asyncio.create_task(
- self._listen_loop(), name="openclaw-ws-listener"
- )
+ self._listener_task = asyncio.create_task(self._listen_loop(), name="openclaw-ws-listener")
return True
else:
err = hello.get("error", {})
@@ -215,6 +222,11 @@ async def connect(self) -> bool:
async def disconnect(self) -> None:
"""Disconnect from the gateway."""
self._connected = False
+ self._use_cli = False
+ self._use_http = False
+ if self._http is not None:
+ await self._http.aclose()
+ self._http = None
if self._listener_task and not self._listener_task.done():
self._listener_task.cancel()
try:
@@ -232,6 +244,29 @@ async def _close_ws(self) -> None:
pass
self._ws = None
+ def _http_base_url(self) -> str:
+ return self.gateway_url.replace("ws://", "http://").replace("wss://", "https://")
+
+ async def _connect_http(self) -> bool:
+ """Prefer the persistent loopback HTTP client over per-turn CLI startup."""
+ headers = {"Authorization": f"Bearer {self.gateway_token}"} if self.gateway_token else {}
+ client = httpx.AsyncClient(
+ base_url=self._http_base_url(),
+ headers=headers,
+ timeout=httpx.Timeout(self.timeout),
+ )
+ try:
+ response = await client.get("/v1/models", timeout=5.0)
+ response.raise_for_status()
+ except Exception:
+ await client.aclose()
+ return False
+ self._http = client
+ self._use_http = True
+ self._connected = True
+ logger.info("Using persistent OpenClaw HTTP chat transport")
+ return True
+
# ------------------------------------------------------------------
# Background listener
# ------------------------------------------------------------------
@@ -284,9 +319,7 @@ async def _dispatch(self, msg: dict) -> None:
# Request helpers
# ------------------------------------------------------------------
- async def _send_request(
- self, method: str, params: dict, timeout: Optional[float] = None
- ) -> dict:
+ async def _send_request(self, method: str, params: dict, timeout: float | None = None) -> dict:
"""Send a request and wait for the response.
Args:
@@ -310,7 +343,7 @@ async def _send_request(
await self._ws.send(json.dumps(req))
result = await asyncio.wait_for(fut, timeout=timeout or self.timeout)
return result
- except asyncio.TimeoutError:
+ except TimeoutError:
self._pending.pop(req_id, None)
return {"ok": False, "error": {"code": "TIMEOUT", "message": "Request timed out"}}
except Exception as e:
@@ -328,8 +361,8 @@ def _full_session_key(self) -> str:
async def chat(
self,
message: str,
- image_b64: Optional[str] = None,
- system_context: Optional[str] = None,
+ image_b64: str | None = None,
+ system_context: str | None = None,
) -> OpenClawResponse:
"""Send a message to OpenClaw and get a response.
@@ -359,6 +392,13 @@ async def chat(
if image_b64:
final_message = f"[Image attached]\n{final_message}"
+ if self._use_http:
+ return await self._chat_via_http(final_message, image_b64=image_b64)
+ if self._use_cli:
+ if image_b64:
+ return await self._chat_with_image_via_gateway_cli(final_message, image_b64)
+ return await self._chat_via_cli(final_message)
+
idempotency_key = str(uuid.uuid4())
session_key = self._full_session_key()
@@ -393,9 +433,7 @@ async def chat(
full_text = ""
while True:
try:
- event = await asyncio.wait_for(
- event_queue.get(), timeout=self.timeout
- )
+ event = await asyncio.wait_for(event_queue.get(), timeout=self.timeout)
payload = event.get("payload", {})
event_name = event.get("event", "")
@@ -425,7 +463,7 @@ async def chat(
full_text = content_parts
break
- except asyncio.TimeoutError:
+ except TimeoutError:
logger.warning("Timeout waiting for chat response (runId=%s)", run_id)
if full_text:
break
@@ -440,10 +478,168 @@ async def chat(
logger.error("OpenClaw chat error: %s", e)
return OpenClawResponse(content="", error=str(e))
+ async def _chat_via_http(
+ self, message: str, image_b64: str | None = None
+ ) -> OpenClawResponse:
+ """Run a normal agent turn without spawning the OpenClaw CLI."""
+ if self._http is None:
+ return OpenClawResponse(content="", error="HTTP transport is unavailable")
+ content: str | list[dict[str, object]] = message
+ if image_b64:
+ content = [
+ {"type": "text", "text": message},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"},
+ },
+ ]
+ try:
+ response = await self._http.post(
+ "/v1/chat/completions",
+ headers={"x-openclaw-session-key": self._full_session_key()},
+ json={
+ "model": f"openclaw/{self.agent_id}",
+ "messages": [{"role": "user", "content": content}],
+ "max_completion_tokens": 180,
+ },
+ )
+ response.raise_for_status()
+ payload = response.json()
+ text = payload["choices"][0]["message"].get("content") or ""
+ return OpenClawResponse(content=text)
+ except Exception as exc:
+ logger.error("OpenClaw HTTP chat failed: %s", exc)
+ return OpenClawResponse(content="", error=str(exc))
+
+ async def _chat_via_cli(self, message: str) -> OpenClawResponse:
+ """Run an agent turn through OpenClaw's supported local CLI client."""
+ process: asyncio.subprocess.Process | None = None
+ command = [
+ "openclaw",
+ "agent",
+ "--agent",
+ self.agent_id,
+ "--session-key",
+ self._full_session_key(),
+ "--message",
+ message,
+ "--thinking",
+ self.thinking,
+ "--timeout",
+ str(max(1, int(self.timeout))),
+ "--json",
+ ]
+ try:
+ process = await asyncio.create_subprocess_exec(
+ *command,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=self.timeout + 15)
+ except TimeoutError:
+ if process is not None and process.returncode is None:
+ process.kill()
+ await process.wait()
+ return OpenClawResponse(content="", error="Response timeout")
+ except Exception as exc:
+ return OpenClawResponse(content="", error=str(exc))
+
+ if process.returncode != 0:
+ detail = stderr.decode("utf-8", errors="replace").strip()
+ return OpenClawResponse(content="", error=detail or f"openclaw exited {process.returncode}")
+
+ try:
+ result = json.loads(stdout)
+ payload = result.get("result") or {}
+ text = payload.get("finalAssistantVisibleText")
+ if not text:
+ chunks = [item.get("text", "") for item in payload.get("payloads", []) if isinstance(item, dict)]
+ text = "\n".join(chunk for chunk in chunks if chunk)
+ return OpenClawResponse(content=text or "")
+ except (json.JSONDecodeError, TypeError) as exc:
+ return OpenClawResponse(content="", error=f"Invalid OpenClaw response: {exc}")
+
+ async def _run_gateway_call(self, method: str, params: dict, timeout_ms: int) -> dict:
+ process = await asyncio.create_subprocess_exec(
+ "openclaw",
+ "gateway",
+ "call",
+ method,
+ "--params",
+ json.dumps(params),
+ "--timeout",
+ str(timeout_ms),
+ "--json",
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ stdout, stderr = await asyncio.wait_for(
+ process.communicate(), timeout=(timeout_ms / 1000) + 15
+ )
+ if process.returncode != 0:
+ detail = stderr.decode("utf-8", errors="replace").strip()
+ raise RuntimeError(detail or f"gateway call exited {process.returncode}")
+ return json.loads(stdout)
+
+ async def _chat_with_image_via_gateway_cli(
+ self, message: str, image_b64: str
+ ) -> OpenClawResponse:
+ """Use authenticated gateway RPC so native image attachments reach the agent."""
+ run_id = str(uuid.uuid4())
+ timeout_ms = max(1_000, int(self.timeout * 1_000))
+ try:
+ started = await self._run_gateway_call(
+ "chat.send",
+ {
+ "sessionKey": self._full_session_key(),
+ "agentId": self.agent_id,
+ "message": message,
+ "attachments": [
+ {
+ "type": "image",
+ "mimeType": "image/jpeg",
+ "fileName": "reachy-view.jpg",
+ "content": image_b64,
+ }
+ ],
+ "idempotencyKey": run_id,
+ },
+ timeout_ms,
+ )
+ actual_run_id = started.get("runId") or run_id
+ await self._run_gateway_call(
+ "agent.wait",
+ {"runId": actual_run_id, "timeoutMs": timeout_ms},
+ timeout_ms,
+ )
+ history = await self._run_gateway_call(
+ "chat.history",
+ {"sessionKey": self._full_session_key(), "limit": 6},
+ 30_000,
+ )
+ for item in reversed(history.get("messages", [])):
+ if item.get("role") != "assistant":
+ continue
+ content = item.get("content", "")
+ if isinstance(content, str):
+ return OpenClawResponse(content=content)
+ if isinstance(content, list):
+ text = "\n".join(
+ part.get("text", "")
+ for part in content
+ if isinstance(part, dict) and part.get("type") == "text"
+ ).strip()
+ if text:
+ return OpenClawResponse(content=text)
+ return OpenClawResponse(content="", error="No assistant response in chat history")
+ except Exception as exc:
+ logger.error("OpenClaw image turn failed: %s", exc)
+ return OpenClawResponse(content="", error=str(exc))
+
async def stream_chat(
self,
message: str,
- image_b64: Optional[str] = None,
+ image_b64: str | None = None,
) -> AsyncIterator[str]:
"""Stream a response from OpenClaw.
@@ -458,6 +654,14 @@ async def stream_chat(
yield "[Error: Not connected to OpenClaw]"
return
+ if self._use_http or self._use_cli:
+ response = await self.chat(message, image_b64=image_b64)
+ if response.error:
+ yield f"[Error: {response.error}]"
+ elif response.content:
+ yield response.content
+ return
+
final_message = message
if image_b64:
final_message = f"[Image attached]\n{message}"
@@ -485,12 +689,9 @@ async def stream_chat(
self._run_events[run_id] = event_queue
try:
- prev_text = ""
while True:
try:
- event = await asyncio.wait_for(
- event_queue.get(), timeout=self.timeout
- )
+ event = await asyncio.wait_for(event_queue.get(), timeout=self.timeout)
payload = event.get("payload", {})
event_name = event.get("event", "")
@@ -509,7 +710,7 @@ async def stream_chat(
elif event_name == "chat" and payload.get("state") == "final":
break
- except asyncio.TimeoutError:
+ except TimeoutError:
yield "[Error: timeout]"
break
finally:
@@ -524,7 +725,7 @@ def is_connected(self) -> bool:
"""Check if bridge is connected to gateway."""
return self._connected
- async def get_agent_context(self) -> Optional[str]:
+ async def get_agent_context(self) -> str | None:
"""Fetch the agent's current context, personality, and memory summary.
This asks OpenClaw to provide a summary of:
@@ -570,9 +771,7 @@ async def get_agent_context(self) -> Optional[str]:
logger.error("Failed to get agent context: %s", e)
return None
- async def sync_conversation(
- self, user_message: str, assistant_response: str
- ) -> None:
+ async def sync_conversation(self, user_message: str, assistant_response: str) -> None:
"""Sync a conversation turn back to OpenClaw for memory continuity.
Args:
@@ -599,7 +798,7 @@ async def sync_conversation(
# Global bridge instance (lazy initialization)
-_bridge: Optional[OpenClawBridge] = None
+_bridge: OpenClawBridge | None = None
def get_bridge() -> OpenClawBridge:
diff --git a/src/reachy_mini_openclaw/prompts.py b/src/reachy_mini_openclaw/prompts.py
index a73a04b..0f9f8dc 100644
--- a/src/reachy_mini_openclaw/prompts.py
+++ b/src/reachy_mini_openclaw/prompts.py
@@ -1,6 +1,6 @@
"""Prompt management for the robot assistant.
-Handles loading and customizing system prompts for the OpenAI Realtime session.
+Handles loading and customizing optional robot profile prompts.
"""
import logging
@@ -16,7 +16,7 @@
def get_session_instructions() -> str:
- """Get the system instructions for the OpenAI Realtime session.
+ """Get the configured robot profile instructions.
Loads from custom profile if configured, otherwise uses default.
@@ -49,16 +49,6 @@ def get_session_instructions() -> str:
Use the camera tool when asked about your surroundings.
Express emotions through movement to enhance communication."""
-
-def get_session_voice() -> str:
- """Get the voice to use for the OpenAI Realtime session.
-
- Returns:
- Voice name string
- """
- return config.OPENAI_VOICE
-
-
def get_available_profiles() -> list[str]:
"""Get list of available prompt profiles.
diff --git a/src/reachy_mini_openclaw/tools/__init__.py b/src/reachy_mini_openclaw/tools/__init__.py
index 56254ca..a25ce8e 100644
--- a/src/reachy_mini_openclaw/tools/__init__.py
+++ b/src/reachy_mini_openclaw/tools/__init__.py
@@ -1,6 +1,6 @@
"""Tool definitions for Reachy Mini OpenClaw.
-These tools are exposed to the OpenAI Realtime API and allow the assistant
+These tools are exposed through the body integration and allow the assistant
to control the robot and interact with the environment.
"""
diff --git a/src/reachy_mini_openclaw/tools/core_tools.py b/src/reachy_mini_openclaw/tools/core_tools.py
index 3028fc3..04a1108 100644
--- a/src/reachy_mini_openclaw/tools/core_tools.py
+++ b/src/reachy_mini_openclaw/tools/core_tools.py
@@ -98,6 +98,7 @@ class ToolDependencies:
camera_worker: Optional[Any] = None
openclaw_bridge: Optional["OpenClawBridge"] = None
vision_manager: Optional[Any] = None # Local vision processor (SmolVLM2)
+ motion_catalog: Optional[Any] = None
# Tool specifications in OpenAI format
@@ -145,15 +146,14 @@ class ToolDependencies:
},
{
"type": "function",
- "name": "dance",
- "description": "Perform a dance animation. Use this to express joy, celebrate, or entertain.",
+ "name": "play_dance",
+ "description": "Play an exact move from Reachy's dynamically discovered dance catalog.",
"parameters": {
"type": "object",
"properties": {
"dance_name": {
"type": "string",
- "enum": ["happy", "excited", "wave", "nod", "shake", "bounce"],
- "description": "The dance to perform"
+ "description": "Exact dance catalog name"
}
},
"required": ["dance_name"]
@@ -161,20 +161,32 @@ class ToolDependencies:
},
{
"type": "function",
- "name": "emotion",
- "description": "Express an emotion through movement. Use this to show reactions and feelings.",
+ "name": "play_emotion",
+ "description": "Play an emotion by exact move name, semantic family, or common alias.",
"parameters": {
"type": "object",
"properties": {
"emotion_name": {
"type": "string",
- "enum": ["happy", "sad", "surprised", "curious", "thinking", "confused", "excited"],
- "description": "The emotion to express"
+ "description": "Exact move (for example proud3), family (proud), or alias (victory)"
}
},
"required": ["emotion_name"]
}
},
+ {
+ "type": "function",
+ "name": "list_moves",
+ "description": "List or search the live Reachy emotion and dance catalogs.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "kind": {"type": "string", "enum": ["emotion", "dance"]},
+ "query": {"type": "string", "description": "Optional case-insensitive name fragment"},
+ },
+ "required": [],
+ },
+ },
{
"type": "function",
"name": "stop_moves",
@@ -231,8 +243,9 @@ async def dispatch_tool_call(
"look": _handle_look,
"camera": _handle_camera,
"face_tracking": _handle_face_tracking,
- "dance": _handle_dance,
- "emotion": _handle_emotion,
+ "play_dance": _handle_dance,
+ "play_emotion": _handle_emotion,
+ "list_moves": _handle_list_moves,
"stop_moves": _handle_stop_moves,
"idle": _handle_idle,
}
@@ -377,8 +390,8 @@ async def _handle_face_tracking(args: dict, deps: ToolDependencies) -> dict:
return {"error": "Camera not available for face tracking"}
try:
- # Check if head tracker is available
- if deps.camera_worker.head_tracker is None:
+ # A detector can live in this process or in Reachy's daemon.
+ if deps.camera_worker.head_tracker is None and not deps.camera_worker.daemon_tracking:
return {"error": "Face tracking not available - no head tracker initialized"}
deps.camera_worker.set_head_tracking_enabled(enabled)
@@ -389,64 +402,55 @@ async def _handle_face_tracking(args: dict, deps: ToolDependencies) -> dict:
async def _handle_dance(args: dict, deps: ToolDependencies) -> dict:
"""Handle dance tool."""
- dance_name = args.get("dance_name", "happy")
-
+ dance_name = args.get("dance_name", "")
try:
- # Try to use dance library if available
- from reachy_mini_dances_library import dances
-
- if hasattr(dances, dance_name):
- dance_class = getattr(dances, dance_name)
- dance_move = dance_class()
- deps.movement_manager.queue_move(dance_move)
- return {"status": "success", "dance": dance_name}
- else:
- # Fallback to simple head movement
- return await _handle_emotion({"emotion_name": dance_name}, deps)
- except ImportError:
- # No dance library, use emotion as fallback
- return await _handle_emotion({"emotion_name": dance_name}, deps)
+ if deps.motion_catalog is None:
+ return {"error": "Motion catalog is unavailable"}
+ deps.movement_manager.queue_move(deps.motion_catalog.get("dance", dance_name))
+ return {"status": "success", "dance": dance_name}
except Exception as e:
return {"error": str(e)}
async def _handle_emotion(args: dict, deps: ToolDependencies) -> dict:
"""Handle emotion expression."""
- from reachy_mini_openclaw.moves import HeadLookMove
-
- emotion_name = args.get("emotion_name", "happy")
-
- # Map emotions to simple head movements
- emotion_sequences = {
- "happy": ["up", "front"],
- "sad": ["down"],
- "surprised": ["up", "front"],
- "curious": ["right", "left", "front"],
- "thinking": ["up", "left"],
- "confused": ["left", "right", "front"],
- "excited": ["up", "down", "up", "front"],
- }
-
- sequence = emotion_sequences.get(emotion_name, ["front"])
-
+ emotion_name = args.get("emotion_name", "")
try:
- for direction in sequence:
- _, current_ant = deps.robot.get_current_joint_positions()
- current_head = deps.robot.get_current_head_pose()
-
- move = HeadLookMove(
- direction=direction,
- start_pose=current_head,
- start_antennas=tuple(current_ant),
- duration=0.5,
- )
- deps.movement_manager.queue_move(move)
-
- return {"status": "success", "emotion": emotion_name}
+ if deps.motion_catalog is None:
+ return {"error": "Motion catalog is unavailable"}
+ resolved = deps.motion_catalog.resolve_emotion(emotion_name)
+ if resolved is None:
+ return {"error": f"Unknown emotion intent: {emotion_name}"}
+ deps.movement_manager.queue_move(deps.motion_catalog.get("emotion", resolved))
+ return {"status": "success", "emotion": resolved, "requested": emotion_name}
except Exception as e:
return {"error": str(e)}
+async def _handle_list_moves(args: dict, deps: ToolDependencies) -> dict:
+ if deps.motion_catalog is None:
+ return {"error": "Motion catalog is unavailable"}
+ kind = args.get("kind")
+ query = args.get("query", "")
+ if query:
+ return {"status": "success", "moves": deps.motion_catalog.search(query, kind)}
+ if kind:
+ result = {"status": "success", "moves": {kind: deps.motion_catalog.list(kind)}}
+ if kind == "emotion":
+ result["emotion_families"] = deps.motion_catalog.emotion_map()
+ result["unmapped_emotions"] = deps.motion_catalog.unmapped_emotions()
+ return result
+ return {
+ "status": "success",
+ "moves": {
+ "emotion": deps.motion_catalog.list("emotion"),
+ "dance": deps.motion_catalog.list("dance"),
+ },
+ "emotion_families": deps.motion_catalog.emotion_map(),
+ "unmapped_emotions": deps.motion_catalog.unmapped_emotions(),
+ }
+
+
async def _handle_stop_moves(args: dict, deps: ToolDependencies) -> dict:
"""Stop all movements."""
deps.movement_manager.clear_move_queue()
diff --git a/tests/test_attention.py b/tests/test_attention.py
new file mode 100644
index 0000000..3a5ae1d
--- /dev/null
+++ b/tests/test_attention.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from reachy_mini_openclaw.attention import TapDetector, format_attention
+
+
+def imu(acceleration: list[float], gyroscope: list[float] | None = None) -> dict:
+ return {
+ "accelerometer": acceleration,
+ "gyroscope": gyroscope or [0.0, 0.0, 0.0],
+ }
+
+
+def test_double_tap_detector_requires_two_distinct_impacts() -> None:
+ detector = TapDetector(acceleration_delta=2.0)
+ assert not detector.feed(imu([0.0, 0.0, 9.8]), 0.0)
+ assert not detector.feed(imu([0.0, 0.0, 13.0]), 1.0)
+ assert not detector.feed(imu([0.0, 0.0, 13.0]), 1.05)
+ assert detector.feed(imu([0.0, 0.0, 13.0]), 1.35)
+
+
+def test_taps_too_far_apart_do_not_trigger() -> None:
+ detector = TapDetector(acceleration_delta=2.0, max_gap=0.8)
+ detector.feed(imu([0.0, 0.0, 9.8]), 0.0)
+ assert not detector.feed(imu([0.0, 0.0, 13.0]), 1.0)
+ assert not detector.feed(imu([0.0, 0.0, 13.0]), 2.0)
+
+
+def test_gyroscope_can_detect_deliberate_pat() -> None:
+ detector = TapDetector(gyroscope_threshold=0.5)
+ detector.feed(imu([0.0, 0.0, 9.8]), 0.0)
+ assert not detector.feed(imu([0.0, 0.0, 9.8], [0.7, 0.0, 0.0]), 1.0)
+ assert detector.feed(imu([0.0, 0.0, 9.8], [0.7, 0.0, 0.0]), 1.4)
+
+
+def test_attention_summary_reads_both_sections() -> None:
+ text = format_attention(
+ {
+ "needsAttention": [
+ {
+ "source": "gmail",
+ "sender": "Security",
+ "subject": "New login",
+ },
+ {
+ "source": "annexus",
+ "sender": "Jules",
+ "subject": "Quote status",
+ },
+ ],
+ "actionsTaken": ["Marked 3 low-priority Gmail messages as read."],
+ }
+ )
+ assert "Gmail, from Security: New login" in text
+ assert "Annexus, from Jules: Quote status" in text
+ assert "Actions taken" in text
+ assert "Marked 3" in text
diff --git a/tests/test_conversation.py b/tests/test_conversation.py
new file mode 100644
index 0000000..dcbff7c
--- /dev/null
+++ b/tests/test_conversation.py
@@ -0,0 +1,377 @@
+import asyncio
+import numpy as np
+import pytest
+import threading
+
+from reachy_mini_openclaw.audio.conversation import (
+ ConversationHandler,
+ EnergyVAD,
+ Utterance,
+ command_after_wake_phrase,
+ needs_vision,
+)
+from reachy_mini_openclaw.audio.providers import SynthesizedAudio
+from reachy_mini_openclaw.openclaw_bridge import OpenClawResponse
+
+
+def test_vad_segments_one_utterance_with_prefix() -> None:
+ vad = EnergyVAD(
+ rms_threshold=0.05,
+ activation_ms=50,
+ prefix_ms=100,
+ silence_ms=100,
+ min_speech_ms=100,
+ max_speech_seconds=2,
+ )
+ sample_rate = 1000
+ assert vad.feed(sample_rate, np.zeros(50, dtype=np.float32)) is None
+ assert vad.feed(sample_rate, np.ones(100, dtype=np.float32) * 0.2) is None
+ utterance = vad.feed(sample_rate, np.zeros(100, dtype=np.float32))
+ assert utterance is not None
+ assert utterance.sample_rate == sample_rate
+ assert utterance.samples.size == 200
+ assert not vad.speaking
+
+
+def test_vad_discards_short_noise() -> None:
+ vad = EnergyVAD(rms_threshold=0.05, activation_ms=50, silence_ms=50, min_speech_ms=100)
+ assert vad.feed(1000, np.ones(20, dtype=np.float32)) is None
+ assert vad.feed(1000, np.zeros(50, dtype=np.float32)) is None
+ assert not vad.speaking
+
+
+def test_vad_requires_sustained_loud_onset() -> None:
+ vad = EnergyVAD(
+ rms_threshold=0.05,
+ activation_ms=100,
+ silence_ms=50,
+ min_speech_ms=100,
+ )
+ assert vad.feed(1000, np.ones(40, dtype=np.float32) * 0.2) is None
+ assert not vad.speaking
+ assert vad.feed(1000, np.zeros(20, dtype=np.float32)) is None
+ assert vad.feed(1000, np.ones(60, dtype=np.float32) * 0.2) is None
+ assert not vad.speaking
+ assert vad.feed(1000, np.ones(40, dtype=np.float32) * 0.2) is None
+ assert vad.speaking
+
+
+def test_vad_hardware_gate_blocks_onset_but_not_active_speech() -> None:
+ vad = EnergyVAD(rms_threshold=0.05, activation_ms=50)
+ loud = np.ones(50, dtype=np.float32) * 0.2
+ assert vad.feed(1000, loud, allow_start=False) is None
+ assert not vad.speaking
+ assert vad.feed(1000, loud, allow_start=True) is None
+ assert vad.speaking
+ assert vad.feed(1000, loud, allow_start=False) is None
+ assert vad.speaking
+
+
+def test_visual_questions_request_a_camera_frame() -> None:
+ assert needs_vision("Asmo, can you see me?")
+ assert needs_vision("Look at what I'm holding")
+ assert not needs_vision("I see what you mean")
+
+
+def test_wake_phrase_matching_is_prefix_only_and_punctuation_tolerant() -> None:
+ assert command_after_wake_phrase("Hey, Asmo! What time is it?", "Hey Asmo") == "What time is it?"
+ assert command_after_wake_phrase("hey asmo", "Hey Asmo") == ""
+ assert command_after_wake_phrase("Someone said hey Asmo", "Hey Asmo") is None
+ assert command_after_wake_phrase("Hey Asmodeus", "Hey Asmo") is None
+
+
+def test_wake_phrase_tolerates_common_stt_mangling() -> None:
+ assert command_after_wake_phrase("Hey asthma, tell me a joke", "Hey Asmo") == "tell me a joke"
+ assert command_after_wake_phrase("Hi Esmo what time is it", "Hey Asmo") == "what time is it"
+ assert command_after_wake_phrase("They asked me a question", "Hey Asmo") is None
+
+
+class Movement:
+ def __init__(self) -> None:
+ self.processing = []
+ self.listening = []
+
+ def set_processing(self, value: bool) -> None:
+ self.processing.append(value)
+
+ def set_listening(self, value: bool) -> None:
+ self.listening.append(value)
+
+
+class Wobbler:
+ def __init__(self) -> None:
+ self.feeds = 0
+
+ def feed(self, audio: str) -> None:
+ self.feeds += 1
+
+ def reset(self) -> None:
+ pass
+
+
+class Deps:
+ def __init__(self) -> None:
+ self.movement_manager = Movement()
+ self.head_wobbler = Wobbler()
+
+
+class STT:
+ async def transcribe(self, samples, sample_rate) -> str:
+ return "what time is it"
+
+
+class StaticSTT:
+ def __init__(self, transcript: str) -> None:
+ self.transcript = transcript
+
+ async def transcribe(self, samples, sample_rate) -> str:
+ return self.transcript
+
+
+class TTS:
+ async def synthesize(self, text: str) -> SynthesizedAudio:
+ assert text == "It is robot o'clock."
+ return SynthesizedAudio(24000, np.arange(5000, dtype=np.int16))
+
+
+class FlakySTT(STT):
+ def __init__(self) -> None:
+ self.attempts = 0
+
+ async def transcribe(self, samples, sample_rate) -> str:
+ self.attempts += 1
+ if self.attempts < 3:
+ raise ConnectionError("temporary STT outage")
+ return await super().transcribe(samples, sample_rate)
+
+
+class FlakyTTS(TTS):
+ def __init__(self) -> None:
+ self.attempts = 0
+
+ async def synthesize(self, text: str) -> SynthesizedAudio:
+ self.attempts += 1
+ if self.attempts < 3:
+ raise ConnectionError("temporary TTS outage")
+ return await super().synthesize(text)
+
+
+class Bridge:
+ is_connected = True
+
+ async def connect(self) -> bool:
+ return True
+
+ async def chat(
+ self, message: str, image_b64: str | None, system_context: str
+ ) -> OpenClawResponse:
+ assert message == "what time is it"
+ assert image_b64 is None
+ assert "Reachy Mini" in system_context
+ return OpenClawResponse("It is robot o'clock.")
+
+
+class FlakyBridge(Bridge):
+ def __init__(self) -> None:
+ self.is_connected = False
+ self.attempts = 0
+
+ async def connect(self) -> bool:
+ self.attempts += 1
+ if self.attempts < 3:
+ return False
+ self.is_connected = True
+ return True
+
+
+class RecordingBridge:
+ is_connected = True
+
+ def __init__(self) -> None:
+ self.messages: list[str] = []
+
+ async def connect(self) -> bool:
+ return True
+
+ async def chat(
+ self, message: str, image_b64: str | None, system_context: str
+ ) -> OpenClawResponse:
+ self.messages.append(message)
+ return OpenClawResponse("It is robot o'clock.")
+
+
+@pytest.mark.asyncio
+async def test_pipeline_routes_openclaw_response_to_tts_and_audio_queue() -> None:
+ deps = Deps()
+ handler = ConversationHandler(
+ stt=STT(),
+ tts=TTS(),
+ openclaw_bridge=Bridge(),
+ deps=deps,
+ vad=EnergyVAD(),
+ output_chunk_samples=2048,
+ )
+ await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32)))
+
+ outputs = []
+ while not handler.output_queue.empty():
+ outputs.append(handler.output_queue.get_nowait())
+ assert outputs[0] == {"role": "user", "content": "what time is it"}
+ assert outputs[1] == {"role": "assistant", "content": "It is robot o'clock."}
+ audio = [item for item in outputs if isinstance(item, tuple)]
+ assert [chunk.shape[1] for _, chunk in audio] == [2048, 2048, 904]
+ assert deps.head_wobbler.feeds == 0
+ assert deps.movement_manager.processing == [True, False]
+
+
+@pytest.mark.asyncio
+async def test_wake_phrase_gate_strips_phrase_before_openclaw() -> None:
+ deps = Deps()
+ bridge = RecordingBridge()
+ handler = ConversationHandler(
+ stt=StaticSTT("Hey, Asmo! What time is it?"),
+ tts=TTS(),
+ openclaw_bridge=bridge,
+ deps=deps,
+ vad=EnergyVAD(),
+ wake_phrase_enabled=True,
+ wake_phrase="Hey Asmo",
+ )
+
+ await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32)))
+
+ assert bridge.messages == ["What time is it?"]
+ assert handler.output_queue.get_nowait() == {"role": "user", "content": "What time is it?"}
+
+
+@pytest.mark.asyncio
+async def test_wake_phrase_gate_ignores_ambient_transcript() -> None:
+ deps = Deps()
+ bridge = RecordingBridge()
+ handler = ConversationHandler(
+ stt=StaticSTT("the television is still talking"),
+ tts=TTS(),
+ openclaw_bridge=bridge,
+ deps=deps,
+ vad=EnergyVAD(),
+ wake_phrase_enabled=True,
+ wake_phrase="Hey Asmo",
+ )
+
+ await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32)))
+
+ assert bridge.messages == []
+ assert handler.output_queue.empty()
+ assert deps.movement_manager.processing == [True, False]
+
+
+@pytest.mark.asyncio
+async def test_pipeline_retries_transient_stt_and_tts_failures() -> None:
+ deps = Deps()
+ stt = FlakySTT()
+ tts = FlakyTTS()
+ handler = ConversationHandler(
+ stt=stt,
+ tts=tts,
+ openclaw_bridge=Bridge(),
+ deps=deps,
+ vad=EnergyVAD(),
+ )
+
+ await handler._process_utterance(Utterance(16000, np.ones(1600, dtype=np.float32)))
+
+ assert stt.attempts == 3
+ assert tts.attempts == 3
+
+
+@pytest.mark.asyncio
+async def test_pipeline_reconnects_gateway_with_bounded_backoff() -> None:
+ deps = Deps()
+ bridge = FlakyBridge()
+ handler = ConversationHandler(
+ stt=STT(),
+ tts=TTS(),
+ openclaw_bridge=bridge,
+ deps=deps,
+ vad=EnergyVAD(),
+ )
+
+ await handler._ensure_gateway()
+
+ assert bridge.attempts == 3
+ assert bridge.is_connected
+
+
+@pytest.mark.asyncio
+async def test_native_stop_watcher_does_not_spawn_a_blocking_executor_thread() -> None:
+ import reachy_mini_openclaw.main as native_main
+
+ core = native_main.ClawBodyCore.__new__(native_main.ClawBodyCore)
+ core._external_stop_event = threading.Event()
+ core._stop_event = asyncio.Event()
+
+ watcher = asyncio.create_task(core._watch_external_stop())
+ await asyncio.sleep(0)
+ watcher.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await watcher
+
+
+def test_native_wrapper_propagates_failure_after_cleanup(monkeypatch) -> None:
+ import reachy_mini_openclaw.main as native_main
+
+ class FailingCore:
+ stopped = False
+
+ def __init__(self, **kwargs) -> None:
+ pass
+
+ async def run(self) -> None:
+ raise RuntimeError("simulated daemon loss")
+
+ async def stop(self) -> None:
+ self.stopped = True
+
+ monkeypatch.setattr(native_main, "ClawBodyCore", FailingCore)
+ app = native_main.ReachyMiniOpenclaw.__new__(native_main.ReachyMiniOpenclaw)
+
+ with pytest.raises(RuntimeError, match="simulated daemon loss"):
+ app.run(object(), threading.Event())
+
+
+def test_native_config_requires_explicit_service_endpoints() -> None:
+ from reachy_mini_openclaw.config import Config
+
+ config = Config(STT_BASE_URL="", OPENCLAW_GATEWAY_URL="", CHATTERBOX_URL="")
+ errors = config.validate()
+
+ assert "STT_BASE_URL is required" in errors
+ assert "OPENCLAW_GATEWAY_URL is required" in errors
+ assert "CHATTERBOX_URL is required for Chatterbox TTS" in errors
+
+
+def test_native_config_rejects_empty_enabled_wake_phrase() -> None:
+ from reachy_mini_openclaw.config import Config
+
+ config = Config(ENABLE_WAKE_PHRASE=True, WAKE_PHRASE=" ")
+
+ assert "WAKE_PHRASE must not be empty when wake phrase gating is enabled" in config.validate()
+
+
+@pytest.mark.asyncio
+async def test_microphone_is_suppressed_while_robot_audio_is_playing() -> None:
+ deps = Deps()
+ vad = EnergyVAD(rms_threshold=0.01)
+ handler = ConversationHandler(
+ stt=STT(),
+ tts=TTS(),
+ openclaw_bridge=Bridge(),
+ deps=deps,
+ vad=vad,
+ )
+ handler._ignore_input_until = float("inf")
+
+ await handler.receive((16000, np.ones(1600, dtype=np.float32)))
+
+ assert not vad.speaking
+ assert handler._utterances.empty()
diff --git a/tests/test_expressive_motion.py b/tests/test_expressive_motion.py
new file mode 100644
index 0000000..685424d
--- /dev/null
+++ b/tests/test_expressive_motion.py
@@ -0,0 +1,48 @@
+import numpy as np
+from reachy_mini.motion.move import Move
+
+from reachy_mini_openclaw.moves import MovementManager
+
+
+class ExclusiveMove(Move):
+ name = "test_move"
+ exclusive_tracking = True
+
+ @property
+ def duration(self):
+ return 1.0
+
+ def evaluate(self, _t):
+ return np.eye(4), np.zeros(2), 0.0
+
+
+class FakeCamera:
+ def __init__(self):
+ self.is_head_tracking_enabled = True
+ self.changes = []
+
+ def set_head_tracking_enabled(self, enabled):
+ self.is_head_tracking_enabled = enabled
+ self.changes.append(enabled)
+
+ def get_face_tracking_offsets(self):
+ return (0.0, 0.0, 0.0, 0.0, 0.0, 0.5)
+
+
+def test_expressive_move_temporarily_owns_tracking():
+ camera = FakeCamera()
+ manager = MovementManager(object(), camera_worker=camera)
+ manager.queue_move(ExclusiveMove())
+
+ manager._poll_signals(10.0)
+ manager._manage_move_queue(10.0)
+ assert camera.changes == [False]
+
+ # Secondary sound/face/speech offsets must not corrupt choreography.
+ manager.state.face_tracking_offsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.5)
+ _, antennas, yaw = manager._get_secondary_pose()
+ assert antennas == (0.0, 0.0)
+ assert yaw == 0.0
+
+ manager._manage_move_queue(11.1)
+ assert camera.changes == [False, True]
diff --git a/tests/test_motion_catalog.py b/tests/test_motion_catalog.py
new file mode 100644
index 0000000..d9d1cdc
--- /dev/null
+++ b/tests/test_motion_catalog.py
@@ -0,0 +1,121 @@
+from reachy_mini_openclaw.motion_catalog import (
+ EMOTION_FAMILIES,
+ UTILITY_RECORDED_MOVES,
+ MotionCatalog,
+)
+
+
+class FakeMoves:
+ def __init__(self, names):
+ self.names = names
+
+ def list_moves(self):
+ return self.names
+
+ def get(self, name):
+ if name not in self.names:
+ raise ValueError(name)
+ return name
+
+
+def catalog():
+ item = object.__new__(MotionCatalog)
+ item.emotions = FakeMoves(
+ ["thoughtful1", "thoughtful2", "laughing2", "inquiring2", "success2", "anxiety1"]
+ )
+ return item
+
+
+def full_catalog():
+ item = object.__new__(MotionCatalog)
+ expressive = [name for variants in EMOTION_FAMILIES.values() for name in variants]
+ item.emotions = FakeMoves(expressive + list(UTILITY_RECORDED_MOVES))
+ return item
+
+
+def test_parse_and_strip_valid_directive():
+ clean, directive = catalog().parse_response("Let me think. [[emotion:thoughtful2]]")
+ assert clean == "Let me think."
+ assert directive.name == "thoughtful2"
+
+
+def test_invalid_directive_is_removed_but_not_played():
+ clean, directive = catalog().parse_response("Nope. [[dance:table_flip]]")
+ assert clean == "Nope."
+ assert directive is None
+
+
+def test_conservative_question_fallback():
+ directive = catalog().infer("What fresh hell is this?")
+ assert directive.name == "inquiring2"
+
+
+def test_explicit_dance_request_has_deterministic_fallback():
+ directive = catalog().infer_request("Hey Asmo, do a little dance for me")
+ assert directive.kind == "dance"
+ assert directive.name == "side_to_side_sway"
+
+
+def test_negated_dance_request_does_not_move():
+ assert catalog().infer_request("Please don't dance right now") is None
+
+
+def test_spoken_official_dance_name_resolves_exactly():
+ directive = catalog().infer_request("Can you try headbanger combo?")
+ assert directive.kind == "dance"
+ assert directive.name == "headbanger_combo"
+
+
+def test_spoken_emotion_intent_resolves_without_model_directive():
+ directive = catalog().infer_request("Can you express anxiety as an emotion?")
+ assert directive.kind == "emotion"
+ assert directive.name == "anxiety1"
+
+
+def test_console_emotion_map_covers_exactly_81_unique_moves():
+ names = [name for variants in EMOTION_FAMILIES.values() for name in variants]
+ assert len(names) == 81
+ assert len(set(names)) == 81
+
+
+def test_utility_recordings_are_not_presented_as_emotions():
+ item = full_catalog()
+ assert len(item.list("emotion")) == 81
+ assert item.list("utility") == sorted(UTILITY_RECORDED_MOVES)
+ assert not (set(item.list("emotion")) & UTILITY_RECORDED_MOVES)
+
+
+def test_all_installed_emotions_have_semantic_family():
+ item = full_catalog()
+ assert item.unmapped_emotions() == []
+ assert set(item.emotion_map()) == set(EMOTION_FAMILIES)
+
+
+def test_family_and_alias_resolve_to_restrained_default():
+ item = full_catalog()
+ assert item.resolve_emotion("proud") == "proud1"
+ assert item.resolve_emotion("victory") == "success1"
+ assert item.resolve_emotion("grossed out") == "disgusted1"
+
+
+def test_exact_numbered_variant_remains_addressable():
+ assert full_catalog().resolve_emotion("proud three") == "proud3"
+ assert full_catalog().resolve_emotion("proud3") == "proud3"
+
+
+def test_specific_yes_no_moods_resolve():
+ item = full_catalog()
+ assert item.resolve_emotion("sad no") == "no_sad1"
+ assert item.resolve_emotion("excited no") == "no_excited1"
+ assert item.resolve_emotion("sad yes") == "yes_sad1"
+
+
+def test_response_directive_accepts_semantic_family():
+ clean, directive = full_catalog().parse_response("Indeed. [[emotion:proud]]")
+ assert clean == "Indeed."
+ assert directive.name == "proud1"
+
+
+def test_explicit_request_resolves_broad_emotion_aliases():
+ directive = full_catalog().infer_request("Show me how grossed out you are")
+ assert directive == type(directive)("emotion", "disgusted1")
diff --git a/tests/test_playback.py b/tests/test_playback.py
new file mode 100644
index 0000000..1cdcb7e
--- /dev/null
+++ b/tests/test_playback.py
@@ -0,0 +1,16 @@
+import numpy as np
+
+from reachy_mini_openclaw.audio.playback import apply_output_gain
+
+
+def test_output_gain_preserves_full_scale_by_default() -> None:
+ samples = np.array([-1.0, -0.5, 0.5, 1.0], dtype=np.float32)
+ np.testing.assert_allclose(apply_output_gain(samples, 1.0), samples)
+
+
+def test_output_gain_boosts_and_clips_safely() -> None:
+ samples = np.array([-0.75, -0.25, 0.25, 0.75], dtype=np.float32)
+ np.testing.assert_allclose(
+ apply_output_gain(samples, 2.0),
+ np.array([-1.0, -0.5, 0.5, 1.0], dtype=np.float32),
+ )
diff --git a/tests/test_speech_providers.py b/tests/test_speech_providers.py
new file mode 100644
index 0000000..d9197d7
--- /dev/null
+++ b/tests/test_speech_providers.py
@@ -0,0 +1,115 @@
+import io
+import wave
+
+import httpx
+import numpy as np
+import pytest
+
+from reachy_mini_openclaw.audio.providers import (
+ ChatterboxTTS,
+ ElevenLabsTTS,
+ FallbackTTS,
+ OpenAICompatibleSTT,
+ SynthesizedAudio,
+ decode_wav,
+ encode_wav,
+)
+
+
+def test_wav_round_trip() -> None:
+ source = np.linspace(-0.5, 0.5, 800, dtype=np.float32)
+ decoded = decode_wav(encode_wav(source, 16000))
+ assert decoded.sample_rate == 16000
+ assert decoded.samples.dtype == np.int16
+ assert decoded.samples.size == source.size
+
+
+@pytest.mark.asyncio
+async def test_openai_compatible_stt_posts_wav() -> None:
+ async def handle(request: httpx.Request) -> httpx.Response:
+ assert request.url.path == "/v1/audio/transcriptions"
+ assert request.headers["authorization"] == "Bearer local-secret"
+ body = await request.aread()
+ assert b'distil-large-v3' in body
+ assert b'filename="utterance.wav"' in body
+ return httpx.Response(200, json={"text": " hello Reachy "})
+
+ client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+ provider = OpenAICompatibleSTT(
+ "http://stt.test/v1",
+ "distil-large-v3",
+ api_key="local-secret",
+ client=client,
+ )
+ try:
+ text = await provider.transcribe(np.ones(1600, dtype=np.float32) * 0.1, 16000)
+ finally:
+ await client.aclose()
+ assert text == "hello Reachy"
+
+
+def _wav_bytes() -> bytes:
+ output = io.BytesIO()
+ with wave.open(output, "wb") as wav:
+ wav.setnchannels(1)
+ wav.setsampwidth(2)
+ wav.setframerate(24000)
+ wav.writeframes(np.arange(64, dtype=np.int16).tobytes())
+ return output.getvalue()
+
+
+@pytest.mark.asyncio
+async def test_chatterbox_requests_named_voice_and_decodes_wav() -> None:
+ async def handle(request: httpx.Request) -> httpx.Response:
+ assert request.url.path == "/v1/audio/speech"
+ payload = __import__("json").loads((await request.aread()).decode())
+ assert payload == {
+ "input": "hello",
+ "voice": "asmo",
+ "model": "chatterbox-turbo",
+ "response_format": "wav",
+ }
+ return httpx.Response(200, content=_wav_bytes(), headers={"content-type": "audio/wav"})
+
+ client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+ provider = ChatterboxTTS("http://tts.test/v1/audio/speech", voice="asmo", client=client)
+ try:
+ audio = await provider.synthesize("hello")
+ finally:
+ await client.aclose()
+ assert audio.sample_rate == 24000
+ assert audio.samples.size == 64
+
+
+@pytest.mark.asyncio
+async def test_elevenlabs_requests_raw_pcm() -> None:
+ pcm = np.arange(32, dtype=np.int16).tobytes()
+
+ async def handle(request: httpx.Request) -> httpx.Response:
+ assert request.url.path == "/v1/text-to-speech/voice-123/stream"
+ assert request.url.params["output_format"] == "pcm_24000"
+ assert request.headers["xi-api-key"] == "eleven-secret"
+ return httpx.Response(200, content=pcm)
+
+ client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+ provider = ElevenLabsTTS("eleven-secret", "voice-123", base_url="http://eleven.test", client=client)
+ try:
+ audio = await provider.synthesize("hello")
+ finally:
+ await client.aclose()
+ assert audio.sample_rate == 24000
+ assert audio.samples.tolist() == list(range(32))
+
+
+@pytest.mark.asyncio
+async def test_fallback_tts_uses_second_provider() -> None:
+ class Broken:
+ async def synthesize(self, text: str) -> SynthesizedAudio:
+ raise RuntimeError("offline")
+
+ class Working:
+ async def synthesize(self, text: str) -> SynthesizedAudio:
+ return SynthesizedAudio(24000, np.array([1, 2], dtype=np.int16))
+
+ audio = await FallbackTTS(Broken(), Working()).synthesize("hello")
+ assert audio.samples.tolist() == [1, 2]